Created
May 20, 2019 17:41
-
-
Save Highstaker/6976175d4c1e877c9b721aa1016ed665 to your computer and use it in GitHub Desktop.
Just some experiments and attempts to abstract MySQL quering boilerplate away.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"database/sql" | |
"errors" | |
"fmt" | |
"log" | |
"math/rand" | |
"runtime" | |
"strings" | |
"sync" | |
"time" | |
_ "github.com/go-sql-driver/mysql" | |
) | |
const dbAddrress = "root:12345@(172.17.0.3:3306)" | |
const maxDBConnections = 100 | |
//global manager of connections | |
var mainDB *sql.DB | |
func initDatabase() { | |
//creates the manager that handles connections by itself. No need to close it | |
lMainDB, err := sql.Open("mysql", dbAddrress+"/map") | |
if err != nil { | |
log.Fatalln("Could not create map database pool:", err.Error()) | |
} else { | |
mainDB = lMainDB | |
//setting maximum number of database connections | |
mainDB.SetMaxOpenConns(maxDBConnections) | |
//needed to circumvent "unexpected EOF" and "invalid connection" errors as shown in https://github.com/go-sql-driver/mysql/issues/674 | |
//maybe it should be one second? | |
//also, consider db.SetMaxIdleConns(0) if needed | |
mainDB.SetConnMaxLifetime(time.Second * 60) | |
} | |
} | |
var Error = fmt.Println | |
var Warning = fmt.Println | |
type dummyError struct{} | |
func (d dummyError) Error() string { | |
return "Dummy" | |
} | |
var preparedStatements = map[string]*sql.Stmt{} | |
var preparedStatementsMutex = sync.RWMutex{} | |
func getCreateStatement(db *sql.DB, query string) (*sql.Stmt, error) { | |
preparedStatementsMutex.RLock() | |
var stmt *sql.Stmt | |
var present bool | |
if stmt, present = preparedStatements[query]; present { | |
preparedStatementsMutex.RUnlock() | |
fmt.Println("statement exists for", query) //Debug | |
return stmt, nil | |
} | |
//If not present, let's create. | |
fmt.Println("Creating statement for", query) //Debug | |
preparedStatementsMutex.RUnlock() | |
preparedStatementsMutex.Lock() | |
defer preparedStatementsMutex.Unlock() | |
stmt, err := db.Prepare(query) | |
if err != nil { | |
return nil, err | |
} | |
preparedStatements[query] = stmt | |
return stmt, nil | |
} | |
func QueryDBMultiple(db *sql.DB, query string, params []interface{}, storages ...[]interface{}) (counter int, err error) { | |
err = dummyError{} | |
var stmt *sql.Stmt | |
for err != nil { | |
stmt, err = getCreateStatement(db, query) | |
if err != nil { | |
Error("Could not build query statement", query, ":", err.Error()) | |
time.Sleep(time.Second * 1) | |
continue | |
} | |
rows, err := stmt.Query(params...) | |
if err != nil { | |
// query error | |
Error("Could not execute query", query, ":", err.Error()) | |
time.Sleep(time.Second * 1) | |
continue | |
} | |
var lenStorages int | |
if len(storages) > 0 { | |
lenStorages = len(storages[0]) | |
} else { | |
break | |
} | |
for rows.Next() { | |
if counter >= lenStorages { | |
break | |
} | |
resultsAmount := len(storages) | |
dest := make([]interface{}, resultsAmount) | |
for i := 0; i < resultsAmount; i++ { | |
dest[i] = &(storages[i][counter]) | |
} | |
if err = rows.Scan(dest...); err != nil { | |
Error("Could not read query results:", query, ":", err.Error()) | |
rows.Close() | |
time.Sleep(time.Second * 1) | |
continue | |
} | |
counter++ | |
} | |
if err := rows.Err(); err != nil { | |
Error("Could not scan rows fully:", query, ":", err.Error()) | |
rows.Close() | |
time.Sleep(time.Second * 1) | |
continue | |
} | |
rows.Close() | |
} | |
return counter, nil | |
} | |
func queryDBSingular(db *sql.DB, query string, params []interface{}, storages ...interface{}) error { | |
var err error = dummyError{} | |
var stmt *sql.Stmt | |
for err != nil { | |
stmt, err = getCreateStatement(db, query) | |
if err != nil { | |
Error("Could not build query statement", query, ":", err.Error()) | |
time.Sleep(time.Second * 1) | |
continue | |
} | |
// fmt.Println("Quering", query) //debug | |
row := stmt.QueryRow(params...) | |
// fmt.Println("Scanning", query) //debug | |
err = row.Scan(storages...) | |
// fmt.Println("Scanning error", err) //debug | |
if err != nil { | |
if err == sql.ErrNoRows { | |
return sql.ErrNoRows | |
} | |
Error("Could not execute query!", err.Error()) | |
time.Sleep(time.Second * 1) | |
continue | |
} | |
} | |
return nil | |
} | |
func execDB(db *sql.DB, query string, params []interface{}) (affectedRows int64, err error) { | |
err = dummyError{} | |
var stmt *sql.Stmt | |
for err != nil { | |
stmt, err = getCreateStatement(db, query) | |
if err != nil { | |
Error("Could not build query statement", query, ":", err.Error()) | |
time.Sleep(time.Second * 1) | |
continue | |
} | |
var result sql.Result | |
result, err = stmt.Exec(params...) | |
if err != nil { | |
Error("Could not execute query", query, ":", err.Error()) | |
time.Sleep(time.Second * 1) | |
continue | |
} | |
var erro error | |
affectedRows, erro = result.RowsAffected() | |
if erro != nil { | |
//Not ever ydatabase supports this. | |
//Not a problematic error. Will show a message but will not terminate execution. | |
Warning("Could not get amount of affected rows:", erro.Error()) | |
} | |
} | |
return | |
} | |
func QueryDB(db *sql.DB, query string, params []interface{}, storages ...interface{}) (numericResult int64, err error) { | |
if strings.HasPrefix(strings.ToUpper(strings.Trim(query, " \n\r")), "SELECT") { | |
storagesAmount := len(storages) | |
if storagesAmount == 0 { | |
return 0, errors.New("storage variables are required for SELECT statement") | |
} | |
// https://stackoverflow.com/q/40343471/2052138 | |
switch x := storages[0].(type) { | |
case []interface{}: | |
//The provided storages are slices. | |
//We will try fetching several rows. | |
firstStorageLen := len(x) | |
lStorages := make([][]interface{}, storagesAmount) | |
for i, storage := range storages { | |
if ss, ok := storage.([]interface{}); ok { | |
if len(ss) != firstStorageLen { | |
return 0, errors.New("storages are not of the same size") | |
} | |
lStorages[i] = ss | |
} else { | |
return 0, errors.New("not all storages are slices, even though the first one is") | |
} | |
} | |
// if sss, ok := storages.([]interface{}); ok { | |
returnedRows, erro := QueryDBMultiple(db, query, params, lStorages...) | |
numericResult, err = int64(returnedRows), erro | |
// } | |
case interface{}: | |
//The provided storages are not slices. | |
//Therefore we will get only one row. | |
erro := queryDBSingular(db, query, params, storages...) | |
if erro == nil { | |
numericResult = 1 | |
} else if erro == sql.ErrNoRows { | |
err = nil | |
numericResult = 0 | |
} else { | |
err = erro | |
} | |
default: | |
err = fmt.Errorf("unsupported type: %T", x) | |
} | |
} else { | |
numericResult, err = execDB(db, query, params) | |
} | |
return | |
} | |
const ( | |
dbQueryStringInserter = "INSERT INTO pootis VALUES (?,?)" | |
dbQueryStringSelector = "SELECT arg1, arg2 from pootis where arg1=?" | |
dbQueryStringUpdater = "UPDATE pootis SET arg2=? WHERE arg1=42" | |
) | |
func randomUint8() uint8 { | |
return uint8(rand.Int() % 255) | |
} | |
func inserter(done chan<- bool) { | |
for i := 0; i < 5; i++ { | |
//////// | |
//INSERT | |
//////// | |
params := make([]interface{}, 2) | |
params[0], params[1] = randomUint8(), randomUint8() | |
// fmt.Println("Inserting", mainDB.Stats()) //debug | |
rowsInserted, err := QueryDB(mainDB, dbQueryStringInserter, params) | |
if err == nil { | |
fmt.Println("Successfully inserted", rowsInserted, "rows!") | |
} else { | |
Error("Error on insertion!", err.Error()) | |
} | |
fmt.Println() | |
/////////////// | |
//SELECT single | |
/////////////// | |
params = make([]interface{}, 1) | |
params[0] = 0 | |
var result1, result2 uint8 | |
fmt.Println("Selecting single") //debug | |
rowsObtained, err := QueryDB(mainDB, dbQueryStringSelector, params, &result1, &result2) | |
if err == nil { | |
fmt.Println("Successfully obtained", rowsObtained, "rows!") | |
if rowsObtained > 0 { | |
fmt.Println(params, result1, result2) | |
} | |
} else { | |
Error("Error on insertion!", err.Error()) | |
} | |
fmt.Println() | |
///////////////// | |
//SELECT multiple | |
///////////////// | |
const bufferSize = 10 | |
params = make([]interface{}, 1) | |
params[0] = 0 | |
sresult1 := make([]interface{}, bufferSize) | |
sresult2 := make([]interface{}, bufferSize) | |
// fmt.Println("Selecting multiple", mainDB.Stats()) //debug | |
rowsAmount, _ := QueryDB(mainDB, dbQueryStringSelector, params, sresult1, sresult2) | |
sresult1 = sresult1[:rowsAmount] | |
sresult2 = sresult2[:rowsAmount] | |
fmt.Println(params, sresult1, sresult2) | |
// fmt.Printf("%T\n", sresult1[0]) | |
// data := make([]uint8, len(sresult2)) | |
// for i, el := range sresult2 { | |
// if s, ok := el.(int64); ok { | |
// data[i] = uint8(s) | |
// } | |
// } | |
// fmt.Println("data", data, data[0]+data[1]) | |
fmt.Println() | |
//////// | |
//UPDATE | |
//////// | |
params = make([]interface{}, 1) | |
params[0] = randomUint8() | |
// fmt.Println("Updating", mainDB.Stats()) //debug | |
rowsInserted, err = QueryDB(mainDB, dbQueryStringUpdater, params) | |
if err == nil { | |
fmt.Println("Successfully updated", rowsInserted, "rows!") | |
} else { | |
Error("Error on insertion!", err.Error()) | |
} | |
fmt.Println() | |
fmt.Println("////////////////////////////////////////////////////") | |
fmt.Println() | |
} | |
done <- true | |
} | |
func main() { | |
fmt.Println("Version:", runtime.Version()) | |
initDatabase() | |
nRoutines := maxDBConnections * 2 | |
nRoutines = 1 | |
ok := make(chan bool, nRoutines) | |
for i := 0; i < nRoutines; i++ { | |
go inserter(ok) | |
} | |
for i := 0; i < nRoutines; i++ { | |
<-ok | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment