Created
July 12, 2023 16:44
-
-
Save dasl-/339a846261c9cd69b6ece894b6963d1e to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/go/vt/vtgate/vindexes/sqlite_lookup.go b/go/vt/vtgate/vindexes/sqlite_lookup.go | |
index 870fb8be32..f83d4d0f53 100644 | |
--- a/go/vt/vtgate/vindexes/sqlite_lookup.go | |
+++ b/go/vt/vtgate/vindexes/sqlite_lookup.go | |
@@ -29,106 +29,115 @@ import ( | |
"vitess.io/vitess/go/stats" | |
"vitess.io/vitess/go/vt/key" | |
_ "github.com/mattn/go-sqlite3" // sqlite driver | |
) | |
const ( | |
// Timing metric keys | |
connectTimingKey = "Connect" | |
queryTimingKey = "Query" | |
) | |
var ( | |
_ SingleColumn = (*SqliteLookupUnique)(nil) | |
_ Lookup = (*SqliteLookupUnique)(nil) | |
+ initSqliteDbMutex sync.RWMutex | |
// Metrics | |
timings = stats.NewTimings("SqliteLookupTimings", "Sqlite lookup timings", "operation") | |
) | |
func init() { | |
Register("etsy_sqlite_lookup_unique", NewSqliteLookupUnique) | |
} | |
// SqliteLookupUnique defines a vindex that uses a sqlite lookup table. | |
// The table is expected to define the id column as unique. It's | |
// Unique and a Lookup. | |
type SqliteLookupUnique struct { | |
name string | |
db *sql.DB | |
path string | |
table string | |
from string | |
to string | |
cacheSize string | |
preparedSelect *sql.Stmt | |
+ initFinished boolean | |
} | |
// NewSqliteLookupUnique creates a SqliteLookupUnique vindex. | |
// The supplied map has the following required fields: | |
// | |
// path: path to the backing sqlite file. | |
// table: name of the backing table. | |
// from: the column in the table that has the 'from' value of the lookup vindex. | |
// to: the 'to' column name of the table that contains the keyrange or keyspace id. | |
// | |
// The following fields are optional: | |
// | |
// cache_size: changes the maximum number of disk pages sqlite holds in memory at once | |
// | |
// As writes cannot be distributed across all vtgate machines, and writes must obtain | |
// locks on the sqlite db, SQLite lookups are always read only | |
func NewSqliteLookupUnique(name string, m map[string]string) (Vindex, error) { | |
slu := &SqliteLookupUnique{name: name} | |
slu.path = m["path"] | |
slu.table = m["table"] | |
slu.from = m["from"] | |
slu.to = m["to"] | |
if cacheSize, ok := m["cache_size"]; ok { | |
slu.cacheSize = cacheSize | |
} | |
return slu, nil | |
} | |
func (slu *SqliteLookupUnique) initSqliteDb() error { | |
+ initSqliteDbMutex.Lock() | |
+ defer initSqliteDbMutex.Unlock() | |
+ if slu.initFinished == true { | |
+ return nil // another thread raced and already finished initializing the DB | |
+ } | |
+ | |
var err error | |
// Options defined here: https://github.com/mattn/go-sqlite3#connection-string | |
dbDSN := "file:" + slu.path + "?mode=ro&_query_only=true&immutable=true" | |
if len(slu.cacheSize) > 0 { | |
dbDSN += "&_cache_size=" + slu.cacheSize | |
} | |
connectTime := time.Now() | |
db, err := sql.Open("sqlite3", dbDSN) | |
if err != nil { | |
return err | |
} | |
if db != nil { | |
db.SetMaxOpenConns(0) // no maximum | |
db.SetMaxIdleConns(100) | |
db.SetConnMaxIdleTime(10 * time.Minute) | |
db.SetConnMaxLifetime(time.Hour) | |
} | |
slu.db = db | |
timings.Record(connectTimingKey, connectTime) | |
stmt, err := slu.db.Prepare(fmt.Sprintf("select %s, %s from %s where %s = ?", slu.from, slu.to, slu.table, slu.from)) | |
if err != nil { | |
return err | |
} | |
slu.preparedSelect = stmt | |
+ slu.initFinished = true | |
return nil | |
} | |
// String returns the name of the vindex. | |
func (slu *SqliteLookupUnique) String() string { | |
return slu.name | |
} | |
// Cost returns the cost of this vindex as 5. | |
func (slu *SqliteLookupUnique) Cost() int { | |
return 5 | |
} | |
// IsUnique returns true since the Vindex is unique. | |
@@ -180,32 +189,32 @@ func (slu *SqliteLookupUnique) Verify(ctx context.Context, vcursor VCursor, ids | |
out = append(out, bytes.Equal(ksid, ksids[i])) | |
} else { | |
out = append(out, ok) | |
} | |
} | |
return out, err | |
} | |
func (slu *SqliteLookupUnique) retrieveIDKsidMap(ids []sqltypes.Value) (resultMap map[string][]byte, err error) { | |
// Query sqlite database | |
var query string | |
var results *sql.Rows | |
queryStart := time.Now() | |
- if slu.db == nil { | |
- err = slu.initSqliteDb() | |
+ if slu.initSqliteDb == false { | |
+ err = slu.initSqliteDb() // locking is implemented inside initSqliteDb | |
if err != nil { | |
return nil, err | |
} | |
} | |
if len(ids) == 1 { // use prepared statement | |
results, err = slu.preparedSelect.Query(ids[0].ToString()) | |
if err != nil { | |
return nil, err | |
} | |
} else { // do not use prepared statement | |
query = fmt.Sprintf("select %s, %s from %s where %s in (", slu.from, slu.to, slu.table, slu.from) | |
for _, id := range ids { | |
query += id.ToString() + ", " | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment