Created
April 27, 2023 20:05
-
-
Save parj/c308b4756a0609f922fc4d3db4ab469d to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import com.fasterxml.jackson.databind.ObjectMapper; | |
import java.io.File; | |
import java.sql.*; | |
import java.util.ArrayList; | |
import java.util.List; | |
import java.util.concurrent.ExecutorService; | |
import java.util.concurrent.Executors; | |
public class DatabaseToJson { | |
private static final String DB_URL = "jdbc:mysql://localhost:3306/mydatabase"; | |
private static final String DB_USERNAME = "root"; | |
private static final String DB_PASSWORD = "password"; | |
private static final String SQL_QUERY = "SELECT * FROM mytable"; | |
private static final String JSON_FILE_PATH = "/tmp/data.json"; | |
private static final int CHUNK_SIZE = 1000; | |
public static void main(String[] args) throws Exception { | |
Class.forName("com.mysql.jdbc.Driver"); | |
Connection connection = DriverManager.getConnection(DB_URL, DB_USERNAME, DB_PASSWORD); | |
PreparedStatement statement = connection.prepareStatement(SQL_QUERY, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); | |
statement.setFetchSize(Integer.MIN_VALUE); | |
ResultSet resultSet = statement.executeQuery(); | |
ObjectMapper objectMapper = new ObjectMapper(); | |
File jsonFile = new File(JSON_FILE_PATH); | |
ExecutorService executorService = Executors.newFixedThreadPool(10); | |
List<List<Object>> chunks = chunkResultSet(resultSet, CHUNK_SIZE); | |
for (int i = 0; i < chunks.size(); i++) { | |
executorService.execute(() -> { | |
try { | |
List<Object> chunk = chunks.get(i); | |
List<String> jsonStrings = new ArrayList<>(); | |
while (chunk.size() > 0) { | |
Object[] row = chunk.remove(0).toArray(); | |
jsonStrings.add(objectMapper.writeValueAsString(row)); | |
} | |
synchronized (jsonFile) { | |
objectMapper.writeValue(jsonFile, jsonStrings); | |
} | |
} catch (Exception e) { | |
e.printStackTrace(); | |
} | |
}); | |
} | |
executorService.shutdown(); | |
} | |
private static List<List<Object>> chunkResultSet(ResultSet resultSet, int chunkSize) throws SQLException { | |
List<List<Object>> chunks = new ArrayList<>(); | |
List<Object> chunk = new ArrayList<>(); | |
while (resultSet.next()) { | |
if (chunk.size() >= chunkSize) { | |
chunks.add(chunk); | |
chunk = new ArrayList<>(); | |
} | |
ResultSetMetaData metaData = resultSet.getMetaData(); | |
int columnCount = metaData.getColumnCount(); | |
List<Object> row = new ArrayList<>(); | |
for (int i = 1; i <= columnCount; i++) { | |
Object value = resultSet.getObject(i); | |
row.add(value); | |
} | |
chunk.add(row); | |
} | |
if (chunk.size() > 0) { | |
chunks.add(chunk); | |
} | |
return chunks; | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment