spring-batch to fetch PhraseApp translations

BatchConfiguration.java

package hello;

import hello.dto.PostDto;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.json.JacksonJsonObjectReader;
import org.springframework.batch.item.json.JsonItemReader;
import org.springframework.batch.item.json.builder.JsonItemReaderBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.UrlResource;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.StringJoiner;

@Configuration
@EnableBatchProcessing
public class BatchConfiguration {
    private static final Logger log = LoggerFactory.getLogger(BatchConfiguration.class);
    @Autowired
    public JobBuilderFactory jobBuilderFactory;

    @Autowired
    public StepBuilderFactory stepBuilderFactory;

    @Bean
    public JsonItemReader<PostDto> itemReader() throws Exception {
        URL url = new URL(buildUrl());
        HttpURLConnection con = (HttpURLConnection) url.openConnection();
        initConnection(con);

        UrlResource phraseAppResource = new UrlResource(url);
        int responseCode = con.getResponseCode();
        System.out.println("+++++++++ RESPONSE++++++++++++++ : " + responseCode);

        final JsonItemReader<PostDto> jsonReader = new JsonItemReaderBuilder<PostDto>()
                .name("jsonReader")
                .resource(phraseAppResource)
                .jsonObjectReader(new JacksonJsonObjectReader<>(PostDto.class))
                .strict(false)
                .build();

        return jsonReader;
    }

    private void initConnection(HttpURLConnection con) throws IOException {
        String apiToken = "token {phrase app token value}";
        con.setRequestMethod("GET");
        con.setRequestProperty("Content-Type", "application/json");
        con.setRequestProperty("Authorization", apiToken);
        con.connect();
    }

    private String buildUrl() {
        String apiUrl = "https://classic-json-api.herokuapp.com";
        String postsUrl = "posts";
        StringJoiner joiner = new StringJoiner("/");
        joiner.add(apiUrl).add(postsUrl);

        return joiner.toString();
    }

    @Bean
    public ItemWriter<PostDto> itemWriter() {
        return items -> {
            for (PostDto item : items) {
                System.out.println("item = " + item);
            }
        };
    }

    @Bean
    public Job job() throws Exception {
        return jobBuilderFactory.get("job")
                .start(step())
                .build();
    }

    @Bean
    public Step step() throws Exception {
        return stepBuilderFactory.get("step")
                .<PostDto, PostDto>chunk(5)
                .reader(itemReader())
                .writer(itemWriter())
                .build();
    }

    /*@Bean
    public JsonItemReader<TranslationDto> itemReader() throws Exception {
        URL url = new URL(buildUrl());
        HttpURLConnection con = (HttpURLConnection) url.openConnection();
        initConnection(con);

        UrlResource phraseAppResource = new UrlResource(url);
        int responseCode = con.getResponseCode();
        System.out.println("+++++++++ RESPONSE++++++++++++++ : " + responseCode);


        final JsonItemReader<TranslationDto> jsonReader = new JsonItemReaderBuilder<TranslationDto>()
                .name("jsonReader")
                .resource(phraseAppResource)
                .jsonObjectReader(new JacksonJsonObjectReader<>(TranslationDto.class))
                .strict(false)
                .build();

        return jsonReader;
    }

    private void initConnection(HttpURLConnection con) throws IOException {
        String apiToken = "token {phrase app token}";
        con.setRequestMethod("GET");
        con.setRequestProperty("Content-Type", "application/json");
        con.setRequestProperty("Authorization", apiToken);
        con.connect();
    }

    private String buildUrl() {
        String apiUrl = "https://api.phraseapp.com/api/v2/projects";
        String projectId = "{phrase app project id}";
        String translationsUrl = "translations";
        StringJoiner joiner = new StringJoiner("/");
        joiner.add(apiUrl).add(projectId).add(translationsUrl);

        return joiner.toString();
    }

    @Bean
    public ItemWriter<TranslationDto> itemWriter() {
        return items -> {
            for (TranslationDto item : items) {
                System.out.println("item = " + item);
            }
        };
    }

    @Bean
    public Job job() throws Exception {
        log.info("+++++++++++++ importTranslationsJob +++++++++++++++");
        return jobBuilderFactory.get("job")
                .start(step())
                .build();
    }

    @Bean
    public Step step() throws Exception {
        log.info("+++++++++++++ step ++++++++++++++++++");
        return stepBuilderFactory.get("step")
                .<TranslationDto, TranslationDto>chunk(25)
                .reader(itemReader())
                .writer(itemWriter())
                .build();
    }*/
}

KeyDto.java

package hello.dto;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;

@Getter
@Setter
@ToString
@JsonIgnoreProperties(ignoreUnknown = true)
public class KeyDto {
    private String name;
}

LocaleDto.java

package hello.dto;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;

@Getter
@Setter
@ToString
@JsonIgnoreProperties(ignoreUnknown = true)
public class LocaleDto {

    private String name;

    private String code;
}

PostDTo.java

package hello.dto;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;

@Getter
@Setter
@ToString
@JsonIgnoreProperties(ignoreUnknown = true)
public class PostDto {
    private String title;
    private String body;
}

TranslationDto.java

package hello.dto;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;

@Getter
@Setter
@ToString
@JsonIgnoreProperties(ignoreUnknown = true)
public class TranslationDto {

    private String content;
    private LocaleDto locale;
    private KeyDto key;
}

@Benam I explicitly opened JacksonJsonObjectReader instance before passing it to jsonObjectReader:

@Bean
    public JsonItemReader<TranslationDto> itemReader() throws Exception {
        URL url = new URL(buildUrl());
        HttpURLConnection con = (HttpURLConnection) url.openConnection();
        initConnection(con);

        UrlResource resource = new UrlResource(url);
        int responseCode = con.getResponseCode();
        System.out.println("+++++++++ RESPONSE: " + responseCode);

        final JacksonJsonObjectReader<TranslationDto> translationDtoJacksonJsonObjectReader = new JacksonJsonObjectReader<>(TranslationDto.class);
        translationDtoJacksonJsonObjectReader.open(resource);

        final JsonItemReader<TranslationDto> translationJsonItemReader = new JsonItemReaderBuilder<TranslationDto>()
                .name("translationJsonItemReader")
                .resource(resource)
                .jsonObjectReader(translationDtoJacksonJsonObjectReader)
                .strict(false)
                .build();

        return translationJsonItemReader;
    }

It seems like setting the Authorization value has no effects:

private void initConnection(HttpURLConnection con) throws IOException {
        String apiToken = "token 59e0b0....ce";
        con.setRequestMethod("GET");
        con.setRequestProperty("Content-Type", "application/json");
        con.setRequestProperty("Authorization", apiToken);
        con.connect();
    }

And the 401 code error is catched in UrlResource class when calling getInputStream method:

@Override
	public InputStream getInputStream() throws IOException {
		URLConnection con = this.url.openConnection();
		ResourceUtils.useCachesIfNecessary(con);
		try {
			return con.getInputStream();
		}
		catch (IOException ex) {
			// Close the HTTP connection (if applicable).
			if (con instanceof HttpURLConnection) {
				((HttpURLConnection) con).disconnect();
			}
			throw ex;
		}
	}

Outside of Spring Boot project, in a simple Java class, the same connection works just fine.

After a long debugging of 2 different APIs, - one at http schema, another one - at https one, I discovered that in case of https end-point API, the following condition was evaluated to false:

package org.springframework.batch.item.json

public class JsonItemReader<T> extends AbstractItemCountingItemStreamItemReader<T> implements
		ResourceAwareItemReaderItemStream<T> {
...
@Override
	protected void doOpen() throws Exception {
		if (!this.resource.exists()) {
			if (this.strict) {
				throw new IllegalStateException("Input resource must exist (reader is in 'strict' mode)");
			}
			LOGGER.warn("Input resource does not exist " + this.resource.getDescription());
			return;
		}
...

i.e. the resource didn't exist! In case of the same processing but for nother API end-point at http it was evaluated totrueand everything worked. Why so ?

It seems like in AbstractFileResolvingResource, in exists method, line 55, a new connection is opened:

URLConnection con = url.openConnection();

which has has no Authorization values set up, that's why I'm getting 401 response code in the below lines in the same method:

if (httpCon != null) {
  int code = httpCon.getResponseCode();
  if (code == HttpURLConnection.HTTP_OK) {
    return true;
  }
  else if (code == HttpURLConnection.HTTP_NOT_FOUND) {
    return false;
  }
}

if (con.getContentLengthLong() > 0) {
  return true;
}
if (httpCon != null) {
  // No HTTP OK status, and no content-length header: give up
  httpCon.disconnect();
  return false;
}

So even if I set the necessary header value in my BatchConfiguration class, it seems like all of theam are just ignored 😢 Here is wjat the documentation to URL#openConnection says:

A new instance of URLConnection is created every time when invoking the URLStreamHandler.openConnection(URL) method of the protocol handler for this URL.

It should be noted that a URLConnection instance does not establish the actual network connection on creation. This will happen only when calling URLConnection.connect().

I was looking at the same place and came to the same conculsion. Here is how I located the issue:

public static void main(String[] args) throws Exception {

	System.out.println("http:");
	UrlResource urlResource = new UrlResource(new URL("http://api.phraseapp.com/api/v2/formats"));

	boolean exists = urlResource.exists();
	System.out.println("exists = " + exists);
	boolean isReadable = urlResource.isReadable();
	System.out.println("isReadable = " + isReadable);

	// with https
	System.out.println("https:");
	urlResource = new UrlResource(new URL("https://api.phraseapp.com/api/v2/formats"));

	exists = urlResource.exists();
	System.out.println("exists = " + exists);
	isReadable = urlResource.isReadable();
	System.out.println("isReadable = " + isReadable);
}

prints:

http:
exists = true
isReadable = false
https:
exists = false
isReadable = false

So it won't work in both cases as the json reader checks if the resource exists and isReadable when in strict mode (the json reader should be used in strict mode in this case).

On the other hand, UrlResource does open the URL behind the scene, so I don't know how to pass authentication headers to it.

@benas, Can we use another third library to just hit the resource and pass it to JsonItemReader? Javalite Http works pretty well. Or use RestTemplate ? Any other ideas?

Yes, I'm preparing a working sample with your url and will share it with you asap.

Cool, thank you 👍

Here is an example with plain Java net APIs:

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;

import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.json.JacksonJsonObjectReader;
import org.springframework.batch.item.json.JsonItemReader;
import org.springframework.batch.item.json.builder.JsonItemReaderBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.InputStreamResource;

@Configuration
@EnableBatchProcessing
public class MyJob {

	public static void main(String[] args) throws Exception {
		ApplicationContext context = new AnnotationConfigApplicationContext(MyJob.class);
		JobLauncher jobLauncher = context.getBean(JobLauncher.class);
		Job job = context.getBean(Job.class);
		jobLauncher.run(job, new JobParameters());
	}
	
	@Autowired
	private JobBuilderFactory jobs;

	@Autowired
	private StepBuilderFactory steps;

	@Bean(destroyMethod = "close")
	public InputStream urlResource() throws IOException {
		URL url = new URL("https://classic-json-api.herokuapp.com");
		URLConnection urlConnection = url.openConnection();
		// urlConnection.setRequestProperty("", ""); // set auth headers if necessary
		return urlConnection.getInputStream();
	}

	@Bean
	public JsonItemReader<Pojo> itemReader(InputStream urlResource) {
		return new JsonItemReaderBuilder<Pojo>()
				.name("restReader")
				.resource(new InputStreamResource(urlResource))
				.strict(true)
				.jsonObjectReader(new JacksonJsonObjectReader<>(Pojo.class))
				.build();
	}

	@Bean
	public ItemWriter<Pojo> itemWriter() {
		return items -> {
			for (Pojo item : items) {
				System.out.println("item = " + item.getTitle());
			}
		};
	}

	@Bean
	public Step step() {
		return steps.get("step")
				.<Pojo, Pojo>chunk(5)
				.reader(itemReader(null))
				.writer(itemWriter())
				.build();
	}

	@Bean
	public Job job() {
		return jobs.get("job")
				.start(step())
				.build();
	}

	@JsonIgnoreProperties(ignoreUnknown = true)
	public static class Pojo {

		private String title;

		public Pojo() {
		}

		public String getTitle() {
			return title;
		}

		public void setTitle(String title) {
			this.title = title;
		}
	}

}

prints:

[warn 2019/07/04 11:50:39.925 CEST <main> tid=0x1] No datasource was provided...using a Map based JobRepository

[warn 2019/07/04 11:50:39.930 CEST <main> tid=0x1] No transaction manager was provided, using a ResourcelessTransactionManager

[info 2019/07/04 11:50:40.077 CEST <main> tid=0x1] No TaskExecutor has been set, defaulting to synchronous executor.

[info 2019/07/04 11:50:40.114 CEST <main> tid=0x1] Job: [SimpleJob: [name=job]] launched with the following parameters: [{}]

[info 2019/07/04 11:50:40.159 CEST <main> tid=0x1] Executing step: [step]

item = title-0
item = title-1
item = title-2
item = title-3
item = title-4
item = title-5
item = title-6
item = title-7
item = title-8
item = title-9
[info 2019/07/04 11:50:40.243 CEST <main> tid=0x1] Step: [step] executed in 82ms

[info 2019/07/04 11:50:40.250 CEST <main> tid=0x1] Job: [SimpleJob: [name=job]] completed with the following parameters: [{}] and the following status: [COMPLETED] in 103ms

Let me know if it helps.

@benas, yep, it works now, 🎉 , thank you. Using

.reader(itemReader(null))

looks a little bit weird though.

Great! Glad to help.

For the .reader(itemReader(null)), you can do something like:

	@Bean
	public JsonItemReader<Pojo> itemReader() throws IOException {
		return new JsonItemReaderBuilder<Pojo>()
				.name("restReader")
				.resource(new InputStreamResource(urlResource()))
				.strict(true)
				.jsonObjectReader(new JacksonJsonObjectReader<>(Pojo.class))
				.build();
	}

then remove the null from the reader definition in the step. However, you might need to propagate the exception declaration in all methods with this approach.

May I ask for my bounty 🙃

Cool, it was just a prototype to see how things work together, I'll refactor all this later, thank you.
Sure, I'll update the SO question and apply you the bounty 😄

@benas I can attribute the bounty only in 8 hours. No worries, I shall not forget ...

No worries. The most important thing is to be able to help you!