Last active
July 12, 2023 16:48
-
-
Save dasl-/29f0ee159a97d8f18e67286b54406bf6 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/go/vt/vtgate/vindexes/sqlite_lookup.go b/go/vt/vtgate/vindexes/sqlite_lookup.go | |
index 870fb8be32..ff6df7f31b 100644 | |
--- a/go/vt/vtgate/vindexes/sqlite_lookup.go | |
+++ b/go/vt/vtgate/vindexes/sqlite_lookup.go | |
@@ -29,106 +29,115 @@ import ( | |
"vitess.io/vitess/go/stats" | |
"vitess.io/vitess/go/vt/key" | |
_ "github.com/mattn/go-sqlite3" // sqlite driver | |
) | |
const ( | |
// Timing metric keys | |
connectTimingKey = "Connect" | |
queryTimingKey = "Query" | |
) | |
var ( | |
_ SingleColumn = (*SqliteLookupUnique)(nil) | |
_ Lookup = (*SqliteLookupUnique)(nil) | |
+ initSqliteDbMutex sync.RWMutex | |
// Metrics | |
timings = stats.NewTimings("SqliteLookupTimings", "Sqlite lookup timings", "operation") | |
) | |
func init() { | |
Register("etsy_sqlite_lookup_unique", NewSqliteLookupUnique) | |
} | |
// SqliteLookupUnique defines a vindex that uses a sqlite lookup table. | |
// The table is expected to define the id column as unique. It's | |
// Unique and a Lookup. | |
type SqliteLookupUnique struct { | |
name string | |
db *sql.DB | |
path string | |
table string | |
from string | |
to string | |
cacheSize string | |
preparedSelect *sql.Stmt | |
+ initFinished boolean | |
} | |
// NewSqliteLookupUnique creates a SqliteLookupUnique vindex. | |
// The supplied map has the following required fields: | |
// | |
// path: path to the backing sqlite file. | |
// table: name of the backing table. | |
// from: the column in the table that has the 'from' value of the lookup vindex. | |
// to: the 'to' column name of the table that contains the keyrange or keyspace id. | |
// | |
// The following fields are optional: | |
// | |
// cache_size: changes the maximum number of disk pages sqlite holds in memory at once | |
// | |
// As writes cannot be distributed across all vtgate machines, and writes must obtain | |
// locks on the sqlite db, SQLite lookups are always read only | |
func NewSqliteLookupUnique(name string, m map[string]string) (Vindex, error) { | |
slu := &SqliteLookupUnique{name: name} | |
slu.path = m["path"] | |
slu.table = m["table"] | |
slu.from = m["from"] | |
slu.to = m["to"] | |
if cacheSize, ok := m["cache_size"]; ok { | |
slu.cacheSize = cacheSize | |
} | |
return slu, nil | |
} | |
func (slu *SqliteLookupUnique) initSqliteDb() error { | |
+ initSqliteDbMutex.Lock() | |
+ defer initSqliteDbMutex.Unlock() | |
+ if slu.initFinished == true { | |
+ return nil // another thread raced and already finished initializing the DB | |
+ } | |
+ | |
var err error | |
// Options defined here: https://github.com/mattn/go-sqlite3#connection-string | |
dbDSN := "file:" + slu.path + "?mode=ro&_query_only=true&immutable=true" | |
if len(slu.cacheSize) > 0 { | |
dbDSN += "&_cache_size=" + slu.cacheSize | |
} | |
connectTime := time.Now() | |
db, err := sql.Open("sqlite3", dbDSN) | |
if err != nil { | |
return err | |
} | |
if db != nil { | |
db.SetMaxOpenConns(0) // no maximum | |
db.SetMaxIdleConns(100) | |
db.SetConnMaxIdleTime(10 * time.Minute) | |
db.SetConnMaxLifetime(time.Hour) | |
} | |
slu.db = db | |
timings.Record(connectTimingKey, connectTime) | |
stmt, err := slu.db.Prepare(fmt.Sprintf("select %s, %s from %s where %s = ?", slu.from, slu.to, slu.table, slu.from)) | |
if err != nil { | |
return err | |
} | |
slu.preparedSelect = stmt | |
+ slu.initFinished = true | |
return nil | |
} | |
// String returns the name of the vindex. | |
func (slu *SqliteLookupUnique) String() string { | |
return slu.name | |
} | |
// Cost returns the cost of this vindex as 5. | |
func (slu *SqliteLookupUnique) Cost() int { | |
return 5 | |
} | |
// IsUnique returns true since the Vindex is unique. | |
@@ -180,31 +189,31 @@ func (slu *SqliteLookupUnique) Verify(ctx context.Context, vcursor VCursor, ids | |
out = append(out, bytes.Equal(ksid, ksids[i])) | |
} else { | |
out = append(out, ok) | |
} | |
} | |
return out, err | |
} | |
func (slu *SqliteLookupUnique) retrieveIDKsidMap(ids []sqltypes.Value) (resultMap map[string][]byte, err error) { | |
// Query sqlite database | |
var query string | |
var results *sql.Rows | |
queryStart := time.Now() | |
- if slu.db == nil { | |
+ if slu.initFinished == false { | |
err = slu.initSqliteDb() | |
if err != nil { | |
return nil, err | |
} | |
} | |
if len(ids) == 1 { // use prepared statement | |
results, err = slu.preparedSelect.Query(ids[0].ToString()) | |
if err != nil { | |
return nil, err | |
} | |
} else { // do not use prepared statement | |
query = fmt.Sprintf("select %s, %s from %s where %s in (", slu.from, slu.to, slu.table, slu.from) | |
for _, id := range ids { | |
query += id.ToString() + ", " |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment