Created
September 24, 2018 13:43
-
-
Save mingodad/2f05cd1280d58f93f89133b2a2011a4d to your computer and use it in GitHub Desktop.
Here is the diffs to an implementation of "notcompressed" option for columns in fts3/4 in sqlite3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Index: ext/fts3/fts3.c | |
================================================================== | |
--- ext/fts3/fts3.c | |
+++ ext/fts3/fts3.c | |
@@ -851,11 +851,12 @@ | |
}else{ | |
zFree = zFunction = fts3QuoteId(zFunc); | |
} | |
fts3Appendf(pRc, &zRet, "docid"); | |
for(i=0; i<p->nColumn; i++){ | |
- fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); | |
+ fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", | |
+ p->abNotcompressed[i] ? "" : zFunction, i, p->azColumn[i]); | |
} | |
if( p->zLanguageid ){ | |
fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); | |
} | |
sqlite3_free(zFree); | |
@@ -907,11 +908,11 @@ | |
}else{ | |
zFree = zFunction = fts3QuoteId(zFunc); | |
} | |
fts3Appendf(pRc, &zRet, "?"); | |
for(i=0; i<p->nColumn; i++){ | |
- fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); | |
+ fts3Appendf(pRc, &zRet, ",%s(?)", p->abNotcompressed[i] ? "" : zFunction); | |
} | |
if( p->zLanguageid ){ | |
fts3Appendf(pRc, &zRet, ", ?"); | |
} | |
sqlite3_free(zFree); | |
@@ -1145,10 +1146,12 @@ | |
char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ | |
char *zContent = 0; /* content=? parameter (or NULL) */ | |
char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ | |
char **azNotindexed = 0; /* The set of notindexed= columns */ | |
int nNotindexed = 0; /* Size of azNotindexed[] array */ | |
+ char **azNotcompressed = 0; /* The set of notcompessed= columns */ | |
+ int nNotcompressed = 0; /* Size of azNotindexed[] array */ | |
assert( strlen(argv[0])==4 ); | |
assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) | |
|| (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) | |
); | |
@@ -1159,15 +1162,19 @@ | |
nByte = sizeof(const char *) * (argc-2); | |
aCol = (const char **)sqlite3_malloc(nByte); | |
if( aCol ){ | |
memset((void*)aCol, 0, nByte); | |
azNotindexed = (char **)sqlite3_malloc(nByte); | |
+ azNotcompressed = (char **)sqlite3_malloc(nByte); | |
} | |
if( azNotindexed ){ | |
memset(azNotindexed, 0, nByte); | |
} | |
- if( !aCol || !azNotindexed ){ | |
+ if( azNotcompressed ){ | |
+ memset(azNotcompressed, 0, nByte); | |
+ } | |
+ if( !aCol || !azNotindexed | !azNotcompressed){ | |
rc = SQLITE_NOMEM; | |
goto fts3_init_out; | |
} | |
/* Loop through all of the arguments passed by the user to the FTS3/4 | |
@@ -1206,11 +1213,12 @@ | |
{ "compress", 8 }, /* 2 -> COMPRESS */ | |
{ "uncompress", 10 }, /* 3 -> UNCOMPRESS */ | |
{ "order", 5 }, /* 4 -> ORDER */ | |
{ "content", 7 }, /* 5 -> CONTENT */ | |
{ "languageid", 10 }, /* 6 -> LANGUAGEID */ | |
- { "notindexed", 10 } /* 7 -> NOTINDEXED */ | |
+ { "notindexed", 10 }, /* 7 -> NOTINDEXED */ | |
+ { "notcompressed", 13 } /* 7 -> NOTCOMPRESSED */ | |
}; | |
int iOpt; | |
if( !zVal ){ | |
rc = SQLITE_NOMEM; | |
@@ -1273,10 +1281,15 @@ | |
case 7: /* NOTINDEXED */ | |
azNotindexed[nNotindexed++] = zVal; | |
zVal = 0; | |
break; | |
+ | |
+ case 8: /* NOTCOMPRESSED */ | |
+ azNotcompressed[nNotcompressed++] = zVal; | |
+ zVal = 0; | |
+ break; | |
default: | |
assert( iOpt==SizeofArray(aFts4Opt) ); | |
sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); | |
rc = SQLITE_ERROR; | |
@@ -1350,10 +1363,11 @@ | |
/* Allocate and populate the Fts3Table structure. */ | |
nByte = sizeof(Fts3Table) + /* Fts3Table */ | |
nCol * sizeof(char *) + /* azColumn */ | |
nIndex * sizeof(struct Fts3Index) + /* aIndex */ | |
nCol * sizeof(u8) + /* abNotindexed */ | |
+ nCol * sizeof(u8) + /* abNotcompressed */ | |
nName + /* zName */ | |
nDb + /* zDb */ | |
nString; /* Space for azColumn strings */ | |
p = (Fts3Table*)sqlite3_malloc(nByte); | |
if( p==0 ){ | |
@@ -1384,13 +1398,14 @@ | |
p->nIndex = nIndex; | |
for(i=0; i<nIndex; i++){ | |
fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1); | |
} | |
p->abNotindexed = (u8 *)&p->aIndex[nIndex]; | |
+ p->abNotcompressed = (u8 *)&p->abNotindexed[nCol]; | |
/* Fill in the zName and zDb fields of the vtab structure. */ | |
- zCsr = (char *)&p->abNotindexed[nCol]; | |
+ zCsr = (char *)&p->abNotcompressed[nCol]; | |
p->zName = zCsr; | |
memcpy(zCsr, argv[2], nName); | |
zCsr += nName; | |
p->zDb = zCsr; | |
memcpy(zCsr, argv[1], nDb); | |
@@ -1429,10 +1444,31 @@ | |
if( azNotindexed[i] ){ | |
sqlite3Fts3ErrMsg(pzErr, "no such column: %s", azNotindexed[i]); | |
rc = SQLITE_ERROR; | |
} | |
} | |
+ | |
+ /* Fill in the abNotcompressed array */ | |
+ for(iCol=0; iCol<nCol; iCol++){ | |
+ int n = (int)strlen(p->azColumn[iCol]); | |
+ for(i=0; i<nNotcompressed; i++){ | |
+ char *zNot = azNotcompressed[i]; | |
+ if( zNot && n==(int)strlen(zNot) | |
+ && 0==sqlite3_strnicmp(p->azColumn[iCol], zNot, n) | |
+ ){ | |
+ p->abNotcompressed[iCol] = 1; | |
+ sqlite3_free(zNot); | |
+ azNotcompressed[i] = 0; | |
+ } | |
+ } | |
+ } | |
+ for(i=0; i<nNotcompressed; i++){ | |
+ if( azNotcompressed[i] ){ | |
+ sqlite3Fts3ErrMsg(pzErr, "no such column: %s", azNotcompressed[i]); | |
+ rc = SQLITE_ERROR; | |
+ } | |
+ } | |
if( rc==SQLITE_OK && (zCompress==0)!=(zUncompress==0) ){ | |
char const *zMiss = (zCompress==0 ? "compress" : "uncompress"); | |
rc = SQLITE_ERROR; | |
sqlite3Fts3ErrMsg(pzErr, "missing %s parameter in fts4 constructor", zMiss); | |
@@ -1469,12 +1505,14 @@ | |
sqlite3_free(zCompress); | |
sqlite3_free(zUncompress); | |
sqlite3_free(zContent); | |
sqlite3_free(zLanguageid); | |
for(i=0; i<nNotindexed; i++) sqlite3_free(azNotindexed[i]); | |
+ for(i=0; i<nNotcompressed; i++) sqlite3_free(azNotcompressed[i]); | |
sqlite3_free((void *)aCol); | |
sqlite3_free((void *)azNotindexed); | |
+ sqlite3_free((void *)azNotcompressed); | |
if( rc!=SQLITE_OK ){ | |
if( p ){ | |
fts3DisconnectMethod((sqlite3_vtab *)p); | |
}else if( pTokenizer ){ | |
pTokenizer->pModule->xDestroy(pTokenizer); | |
@@ -3899,10 +3937,11 @@ | |
void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); | |
#endif | |
#ifdef SQLITE_ENABLE_ICU | |
void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); | |
#endif | |
+void sqlite3Fts3SubLatinTokenizerModule(sqlite3_tokenizer_module const**ppModule); | |
Index: ext/fts3/fts3Int.h | |
================================================================== | |
--- ext/fts3/fts3Int.h | |
+++ ext/fts3/fts3Int.h | |
@@ -218,10 +218,11 @@ | |
const char *zDb; /* logical database name */ | |
const char *zName; /* virtual table name */ | |
int nColumn; /* number of named columns in virtual table */ | |
char **azColumn; /* column names. malloced */ | |
u8 *abNotindexed; /* True for 'notindexed' columns */ | |
+ u8 *abNotcompressed; /* True for 'notcompressed' columns */ | |
sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ | |
char *zContentTbl; /* content=xxx option, or NULL */ | |
char *zLanguageid; /* languageid=xxx option, or NULL */ | |
int nAutoincrmerge; /* Value configured by 'automerge' */ | |
u32 nLeafAdd; /* Number of leaf blocks added this trans */ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment