Skip to content

Instantly share code, notes, and snippets.

@parj
Created April 27, 2023 20:05
Show Gist options
  • Save parj/c308b4756a0609f922fc4d3db4ab469d to your computer and use it in GitHub Desktop.
Save parj/c308b4756a0609f922fc4d3db4ab469d to your computer and use it in GitHub Desktop.
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.File;
import java.sql.*;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class DatabaseToJson {
private static final String DB_URL = "jdbc:mysql://localhost:3306/mydatabase";
private static final String DB_USERNAME = "root";
private static final String DB_PASSWORD = "password";
private static final String SQL_QUERY = "SELECT * FROM mytable";
private static final String JSON_FILE_PATH = "/tmp/data.json";
private static final int CHUNK_SIZE = 1000;
public static void main(String[] args) throws Exception {
Class.forName("com.mysql.jdbc.Driver");
Connection connection = DriverManager.getConnection(DB_URL, DB_USERNAME, DB_PASSWORD);
PreparedStatement statement = connection.prepareStatement(SQL_QUERY, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
statement.setFetchSize(Integer.MIN_VALUE);
ResultSet resultSet = statement.executeQuery();
ObjectMapper objectMapper = new ObjectMapper();
File jsonFile = new File(JSON_FILE_PATH);
ExecutorService executorService = Executors.newFixedThreadPool(10);
List<List<Object>> chunks = chunkResultSet(resultSet, CHUNK_SIZE);
for (int i = 0; i < chunks.size(); i++) {
executorService.execute(() -> {
try {
List<Object> chunk = chunks.get(i);
List<String> jsonStrings = new ArrayList<>();
while (chunk.size() > 0) {
Object[] row = chunk.remove(0).toArray();
jsonStrings.add(objectMapper.writeValueAsString(row));
}
synchronized (jsonFile) {
objectMapper.writeValue(jsonFile, jsonStrings);
}
} catch (Exception e) {
e.printStackTrace();
}
});
}
executorService.shutdown();
}
private static List<List<Object>> chunkResultSet(ResultSet resultSet, int chunkSize) throws SQLException {
List<List<Object>> chunks = new ArrayList<>();
List<Object> chunk = new ArrayList<>();
while (resultSet.next()) {
if (chunk.size() >= chunkSize) {
chunks.add(chunk);
chunk = new ArrayList<>();
}
ResultSetMetaData metaData = resultSet.getMetaData();
int columnCount = metaData.getColumnCount();
List<Object> row = new ArrayList<>();
for (int i = 1; i <= columnCount; i++) {
Object value = resultSet.getObject(i);
row.add(value);
}
chunk.add(row);
}
if (chunk.size() > 0) {
chunks.add(chunk);
}
return chunks;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment