amatch.c 44 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487
  1. /*
  2. ** 2013-03-14
  3. **
  4. ** The author disclaims copyright to this source code. In place of
  5. ** a legal notice, here is a blessing:
  6. **
  7. ** May you do good and not evil.
  8. ** May you find forgiveness for yourself and forgive others.
  9. ** May you share freely, never taking more than you give.
  10. **
  11. *************************************************************************
  12. **
  13. ** This file contains code for a demonstration virtual table that finds
  14. ** "approximate matches" - strings from a finite set that are nearly the
  15. ** same as a single input string. The virtual table is called "amatch".
  16. **
  17. ** A amatch virtual table is created like this:
  18. **
  19. ** CREATE VIRTUAL TABLE f USING approximate_match(
  20. ** vocabulary_table=<tablename>, -- V
  21. ** vocabulary_word=<columnname>, -- W
  22. ** vocabulary_language=<columnname>, -- L
  23. ** edit_distances=<edit-cost-table>
  24. ** );
  25. **
  26. ** When it is created, the new amatch table must be supplied with the
  27. ** the name of a table V and columns V.W and V.L such that
  28. **
  29. ** SELECT W FROM V WHERE L=$language
  30. **
  31. ** returns the allowed vocabulary for the match. If the "vocabulary_language"
  32. ** or L columnname is left unspecified or is an empty string, then no
  33. ** filtering of the vocabulary by language is performed.
  34. **
  35. ** For efficiency, it is essential that the vocabulary table be indexed:
  36. **
  37. ** CREATE vocab_index ON V(W)
  38. **
  39. ** A separate edit-cost-table provides scoring information that defines
  40. ** what it means for one string to be "close" to another.
  41. **
  42. ** The edit-cost-table must contain exactly four columns (more precisely,
  43. ** the statement "SELECT * FROM <edit-cost-table>" must return records
  44. ** that consist of four columns). It does not matter what the columns are
  45. ** named.
  46. **
  47. ** Each row in the edit-cost-table represents a single character
  48. ** transformation going from user input to the vocabulary. The leftmost
  49. ** column of the row (column 0) contains an integer identifier of the
  50. ** language to which the transformation rule belongs (see "MULTIPLE LANGUAGES"
  51. ** below). The second column of the row (column 1) contains the input
  52. ** character or characters - the characters of user input. The third
  53. ** column contains characters as they appear in the vocabulary table.
  54. ** And the fourth column contains the integer cost of making the
  55. ** transformation. For example:
  56. **
  57. ** CREATE TABLE f_data(iLang, cFrom, cTo, Cost);
  58. ** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '', 'a', 100);
  59. ** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'b', '', 87);
  60. ** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'o', 'oe', 38);
  61. ** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'oe', 'o', 40);
  62. **
  63. ** The first row inserted into the edit-cost-table by the SQL script
  64. ** above indicates that the cost of having an extra 'a' in the vocabulary
  65. ** table that is missing in the user input 100. (All costs are integers.
  66. ** Overall cost must not exceed 16777216.) The second INSERT statement
  67. ** creates a rule saying that the cost of having a single letter 'b' in
  68. ** user input which is missing in the vocabulary table is 87. The third
  69. ** INSERT statement mean that the cost of matching an 'o' in user input
  70. ** against an 'oe' in the vocabulary table is 38. And so forth.
  71. **
  72. ** The following rules are special:
  73. **
  74. ** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '?', '', 97);
  75. ** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '', '?', 98);
  76. ** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '?', '?', 99);
  77. **
  78. ** The '?' to '' rule is the cost of having any single character in the input
  79. ** that is not found in the vocabular. The '' to '?' rule is the cost of
  80. ** having a character in the vocabulary table that is missing from input.
  81. ** And the '?' to '?' rule is the cost of doing an arbitrary character
  82. ** substitution. These three generic rules apply across all languages.
  83. ** In other words, the iLang field is ignored for the generic substitution
  84. ** rules. If more than one cost is given for a generic substitution rule,
  85. ** then the lowest cost is used.
  86. **
  87. ** Once it has been created, the amatch virtual table can be queried
  88. ** as follows:
  89. **
  90. ** SELECT word, distance FROM f
  91. ** WHERE word MATCH 'abcdefg'
  92. ** AND distance<200;
  93. **
  94. ** This query outputs the strings contained in the T(F) field that
  95. ** are close to "abcdefg" and in order of increasing distance. No string
  96. ** is output more than once. If there are multiple ways to transform the
  97. ** target string ("abcdefg") into a string in the vocabulary table then
  98. ** the lowest cost transform is the one that is returned. In this example,
  99. ** the search is limited to strings with a total distance of less than 200.
  100. **
  101. ** For efficiency, it is important to put tight bounds on the distance.
  102. ** The time and memory space needed to perform this query is exponential
  103. ** in the maximum distance. A good rule of thumb is to limit the distance
  104. ** to no more than 1.5 or 2 times the maximum cost of any rule in the
  105. ** edit-cost-table.
  106. **
  107. ** The amatch is a read-only table. Any attempt to DELETE, INSERT, or
  108. ** UPDATE on a amatch table will throw an error.
  109. **
  110. ** It is important to put some kind of a limit on the amatch output. This
  111. ** can be either in the form of a LIMIT clause at the end of the query,
  112. ** or better, a "distance<NNN" constraint where NNN is some number. The
  113. ** running time and memory requirement is exponential in the value of NNN
  114. ** so you want to make sure that NNN is not too big. A value of NNN that
  115. ** is about twice the average transformation cost seems to give good results.
  116. **
  117. ** The amatch table can be useful for tasks such as spelling correction.
  118. ** Suppose all allowed words are in table vocabulary(w). Then one would create
  119. ** an amatch virtual table like this:
  120. **
  121. ** CREATE VIRTUAL TABLE ex1 USING amatch(
  122. ** vocabtable=vocabulary,
  123. ** vocabcolumn=w,
  124. ** edit_distances=ec1
  125. ** );
  126. **
  127. ** Then given an input word $word, look up close spellings this way:
  128. **
  129. ** SELECT word, distance FROM ex1
  130. ** WHERE word MATCH $word AND distance<200;
  131. **
  132. ** MULTIPLE LANGUAGES
  133. **
  134. ** Normally, the "iLang" value associated with all character transformations
  135. ** in the edit-cost-table is zero. However, if required, the amatch
  136. ** virtual table allows multiple languages to be defined. Each query uses
  137. ** only a single iLang value. This allows, for example, a single
  138. ** amatch table to support multiple languages.
  139. **
  140. ** By default, only the rules with iLang=0 are used. To specify an
  141. ** alternative language, a "language = ?" expression must be added to the
  142. ** WHERE clause of a SELECT, where ? is the integer identifier of the desired
  143. ** language. For example:
  144. **
  145. ** SELECT word, distance FROM ex1
  146. ** WHERE word MATCH $word
  147. ** AND distance<=200
  148. ** AND language=1 -- Specify use language 1 instead of 0
  149. **
  150. ** If no "language = ?" constraint is specified in the WHERE clause, language
  151. ** 0 is used.
  152. **
  153. ** LIMITS
  154. **
  155. ** The maximum language number is 2147483647. The maximum length of either
  156. ** of the strings in the second or third column of the amatch data table
  157. ** is 50 bytes. The maximum cost on a rule is 1000.
  158. */
  159. #include "sqlite3ext.h"
  160. SQLITE_EXTENSION_INIT1
  161. #include <stdlib.h>
  162. #include <string.h>
  163. #include <assert.h>
  164. #include <stdio.h>
  165. #include <ctype.h>
  166. #ifndef SQLITE_OMIT_VIRTUALTABLE
  167. /*
  168. ** Forward declaration of objects used by this implementation
  169. */
  170. typedef struct amatch_vtab amatch_vtab;
  171. typedef struct amatch_cursor amatch_cursor;
  172. typedef struct amatch_rule amatch_rule;
  173. typedef struct amatch_word amatch_word;
  174. typedef struct amatch_avl amatch_avl;
  175. /*****************************************************************************
  176. ** AVL Tree implementation
  177. */
  178. /*
  179. ** Objects that want to be members of the AVL tree should embedded an
  180. ** instance of this structure.
  181. */
  182. struct amatch_avl {
  183. amatch_word *pWord; /* Points to the object being stored in the tree */
  184. char *zKey; /* Key. zero-terminated string. Must be unique */
  185. amatch_avl *pBefore; /* Other elements less than zKey */
  186. amatch_avl *pAfter; /* Other elements greater than zKey */
  187. amatch_avl *pUp; /* Parent element */
  188. short int height; /* Height of this node. Leaf==1 */
  189. short int imbalance; /* Height difference between pBefore and pAfter */
  190. };
  191. /* Recompute the amatch_avl.height and amatch_avl.imbalance fields for p.
  192. ** Assume that the children of p have correct heights.
  193. */
  194. static void amatchAvlRecomputeHeight(amatch_avl *p){
  195. short int hBefore = p->pBefore ? p->pBefore->height : 0;
  196. short int hAfter = p->pAfter ? p->pAfter->height : 0;
  197. p->imbalance = hBefore - hAfter; /* -: pAfter higher. +: pBefore higher */
  198. p->height = (hBefore>hAfter ? hBefore : hAfter)+1;
  199. }
  200. /*
  201. ** P B
  202. ** / \ / \
  203. ** B Z ==> X P
  204. ** / \ / \
  205. ** X Y Y Z
  206. **
  207. */
  208. static amatch_avl *amatchAvlRotateBefore(amatch_avl *pP){
  209. amatch_avl *pB = pP->pBefore;
  210. amatch_avl *pY = pB->pAfter;
  211. pB->pUp = pP->pUp;
  212. pB->pAfter = pP;
  213. pP->pUp = pB;
  214. pP->pBefore = pY;
  215. if( pY ) pY->pUp = pP;
  216. amatchAvlRecomputeHeight(pP);
  217. amatchAvlRecomputeHeight(pB);
  218. return pB;
  219. }
  220. /*
  221. ** P A
  222. ** / \ / \
  223. ** X A ==> P Z
  224. ** / \ / \
  225. ** Y Z X Y
  226. **
  227. */
  228. static amatch_avl *amatchAvlRotateAfter(amatch_avl *pP){
  229. amatch_avl *pA = pP->pAfter;
  230. amatch_avl *pY = pA->pBefore;
  231. pA->pUp = pP->pUp;
  232. pA->pBefore = pP;
  233. pP->pUp = pA;
  234. pP->pAfter = pY;
  235. if( pY ) pY->pUp = pP;
  236. amatchAvlRecomputeHeight(pP);
  237. amatchAvlRecomputeHeight(pA);
  238. return pA;
  239. }
  240. /*
  241. ** Return a pointer to the pBefore or pAfter pointer in the parent
  242. ** of p that points to p. Or if p is the root node, return pp.
  243. */
  244. static amatch_avl **amatchAvlFromPtr(amatch_avl *p, amatch_avl **pp){
  245. amatch_avl *pUp = p->pUp;
  246. if( pUp==0 ) return pp;
  247. if( pUp->pAfter==p ) return &pUp->pAfter;
  248. return &pUp->pBefore;
  249. }
  250. /*
  251. ** Rebalance all nodes starting with p and working up to the root.
  252. ** Return the new root.
  253. */
  254. static amatch_avl *amatchAvlBalance(amatch_avl *p){
  255. amatch_avl *pTop = p;
  256. amatch_avl **pp;
  257. while( p ){
  258. amatchAvlRecomputeHeight(p);
  259. if( p->imbalance>=2 ){
  260. amatch_avl *pB = p->pBefore;
  261. if( pB->imbalance<0 ) p->pBefore = amatchAvlRotateAfter(pB);
  262. pp = amatchAvlFromPtr(p,&p);
  263. p = *pp = amatchAvlRotateBefore(p);
  264. }else if( p->imbalance<=(-2) ){
  265. amatch_avl *pA = p->pAfter;
  266. if( pA->imbalance>0 ) p->pAfter = amatchAvlRotateBefore(pA);
  267. pp = amatchAvlFromPtr(p,&p);
  268. p = *pp = amatchAvlRotateAfter(p);
  269. }
  270. pTop = p;
  271. p = p->pUp;
  272. }
  273. return pTop;
  274. }
  275. /* Search the tree rooted at p for an entry with zKey. Return a pointer
  276. ** to the entry or return NULL.
  277. */
  278. static amatch_avl *amatchAvlSearch(amatch_avl *p, const char *zKey){
  279. int c;
  280. while( p && (c = strcmp(zKey, p->zKey))!=0 ){
  281. p = (c<0) ? p->pBefore : p->pAfter;
  282. }
  283. return p;
  284. }
  285. /* Find the first node (the one with the smallest key).
  286. */
  287. static amatch_avl *amatchAvlFirst(amatch_avl *p){
  288. if( p ) while( p->pBefore ) p = p->pBefore;
  289. return p;
  290. }
  291. #if 0 /* NOT USED */
  292. /* Return the node with the next larger key after p.
  293. */
  294. static amatch_avl *amatchAvlNext(amatch_avl *p){
  295. amatch_avl *pPrev = 0;
  296. while( p && p->pAfter==pPrev ){
  297. pPrev = p;
  298. p = p->pUp;
  299. }
  300. if( p && pPrev==0 ){
  301. p = amatchAvlFirst(p->pAfter);
  302. }
  303. return p;
  304. }
  305. #endif
  306. #if 0 /* NOT USED */
  307. /* Verify AVL tree integrity
  308. */
  309. static int amatchAvlIntegrity(amatch_avl *pHead){
  310. amatch_avl *p;
  311. if( pHead==0 ) return 1;
  312. if( (p = pHead->pBefore)!=0 ){
  313. assert( p->pUp==pHead );
  314. assert( amatchAvlIntegrity(p) );
  315. assert( strcmp(p->zKey, pHead->zKey)<0 );
  316. while( p->pAfter ) p = p->pAfter;
  317. assert( strcmp(p->zKey, pHead->zKey)<0 );
  318. }
  319. if( (p = pHead->pAfter)!=0 ){
  320. assert( p->pUp==pHead );
  321. assert( amatchAvlIntegrity(p) );
  322. assert( strcmp(p->zKey, pHead->zKey)>0 );
  323. p = amatchAvlFirst(p);
  324. assert( strcmp(p->zKey, pHead->zKey)>0 );
  325. }
  326. return 1;
  327. }
  328. static int amatchAvlIntegrity2(amatch_avl *pHead){
  329. amatch_avl *p, *pNext;
  330. for(p=amatchAvlFirst(pHead); p; p=pNext){
  331. pNext = amatchAvlNext(p);
  332. if( pNext==0 ) break;
  333. assert( strcmp(p->zKey, pNext->zKey)<0 );
  334. }
  335. return 1;
  336. }
  337. #endif
  338. /* Insert a new node pNew. Return NULL on success. If the key is not
  339. ** unique, then do not perform the insert but instead leave pNew unchanged
  340. ** and return a pointer to an existing node with the same key.
  341. */
  342. static amatch_avl *amatchAvlInsert(amatch_avl **ppHead, amatch_avl *pNew){
  343. int c;
  344. amatch_avl *p = *ppHead;
  345. if( p==0 ){
  346. p = pNew;
  347. pNew->pUp = 0;
  348. }else{
  349. while( p ){
  350. c = strcmp(pNew->zKey, p->zKey);
  351. if( c<0 ){
  352. if( p->pBefore ){
  353. p = p->pBefore;
  354. }else{
  355. p->pBefore = pNew;
  356. pNew->pUp = p;
  357. break;
  358. }
  359. }else if( c>0 ){
  360. if( p->pAfter ){
  361. p = p->pAfter;
  362. }else{
  363. p->pAfter = pNew;
  364. pNew->pUp = p;
  365. break;
  366. }
  367. }else{
  368. return p;
  369. }
  370. }
  371. }
  372. pNew->pBefore = 0;
  373. pNew->pAfter = 0;
  374. pNew->height = 1;
  375. pNew->imbalance = 0;
  376. *ppHead = amatchAvlBalance(p);
  377. /* assert( amatchAvlIntegrity(*ppHead) ); */
  378. /* assert( amatchAvlIntegrity2(*ppHead) ); */
  379. return 0;
  380. }
  381. /* Remove node pOld from the tree. pOld must be an element of the tree or
  382. ** the AVL tree will become corrupt.
  383. */
  384. static void amatchAvlRemove(amatch_avl **ppHead, amatch_avl *pOld){
  385. amatch_avl **ppParent;
  386. amatch_avl *pBalance;
  387. /* assert( amatchAvlSearch(*ppHead, pOld->zKey)==pOld ); */
  388. ppParent = amatchAvlFromPtr(pOld, ppHead);
  389. if( pOld->pBefore==0 && pOld->pAfter==0 ){
  390. *ppParent = 0;
  391. pBalance = pOld->pUp;
  392. }else if( pOld->pBefore && pOld->pAfter ){
  393. amatch_avl *pX, *pY;
  394. pX = amatchAvlFirst(pOld->pAfter);
  395. *amatchAvlFromPtr(pX, 0) = pX->pAfter;
  396. if( pX->pAfter ) pX->pAfter->pUp = pX->pUp;
  397. pBalance = pX->pUp;
  398. pX->pAfter = pOld->pAfter;
  399. if( pX->pAfter ){
  400. pX->pAfter->pUp = pX;
  401. }else{
  402. assert( pBalance==pOld );
  403. pBalance = pX;
  404. }
  405. pX->pBefore = pY = pOld->pBefore;
  406. if( pY ) pY->pUp = pX;
  407. pX->pUp = pOld->pUp;
  408. *ppParent = pX;
  409. }else if( pOld->pBefore==0 ){
  410. *ppParent = pBalance = pOld->pAfter;
  411. pBalance->pUp = pOld->pUp;
  412. }else if( pOld->pAfter==0 ){
  413. *ppParent = pBalance = pOld->pBefore;
  414. pBalance->pUp = pOld->pUp;
  415. }
  416. *ppHead = amatchAvlBalance(pBalance);
  417. pOld->pUp = 0;
  418. pOld->pBefore = 0;
  419. pOld->pAfter = 0;
  420. /* assert( amatchAvlIntegrity(*ppHead) ); */
  421. /* assert( amatchAvlIntegrity2(*ppHead) ); */
  422. }
  423. /*
  424. ** End of the AVL Tree implementation
  425. ******************************************************************************/
  426. /*
  427. ** Various types.
  428. **
  429. ** amatch_cost is the "cost" of an edit operation.
  430. **
  431. ** amatch_len is the length of a matching string.
  432. **
  433. ** amatch_langid is an ruleset identifier.
  434. */
  435. typedef int amatch_cost;
  436. typedef signed char amatch_len;
  437. typedef int amatch_langid;
  438. /*
  439. ** Limits
  440. */
  441. #define AMATCH_MX_LENGTH 50 /* Maximum length of a rule string */
  442. #define AMATCH_MX_LANGID 2147483647 /* Maximum rule ID */
  443. #define AMATCH_MX_COST 1000 /* Maximum single-rule cost */
  444. /*
  445. ** A match or partial match
  446. */
  447. struct amatch_word {
  448. amatch_word *pNext; /* Next on a list of all amatch_words */
  449. amatch_avl sCost; /* Linkage of this node into the cost tree */
  450. amatch_avl sWord; /* Linkage of this node into the word tree */
  451. amatch_cost rCost; /* Cost of the match so far */
  452. int iSeq; /* Sequence number */
  453. char zCost[10]; /* Cost key (text rendering of rCost) */
  454. short int nMatch; /* Input characters matched */
  455. char zWord[4]; /* Text of the word. Extra space appended as needed */
  456. };
  457. /*
  458. ** Each transformation rule is stored as an instance of this object.
  459. ** All rules are kept on a linked list sorted by rCost.
  460. */
  461. struct amatch_rule {
  462. amatch_rule *pNext; /* Next rule in order of increasing rCost */
  463. char *zFrom; /* Transform from (a string from user input) */
  464. amatch_cost rCost; /* Cost of this transformation */
  465. amatch_langid iLang; /* The langauge to which this rule belongs */
  466. amatch_len nFrom, nTo; /* Length of the zFrom and zTo strings */
  467. char zTo[4]; /* Tranform to V.W value (extra space appended) */
  468. };
  469. /*
  470. ** A amatch virtual-table object
  471. */
  472. struct amatch_vtab {
  473. sqlite3_vtab base; /* Base class - must be first */
  474. char *zClassName; /* Name of this class. Default: "amatch" */
  475. char *zDb; /* Name of database. (ex: "main") */
  476. char *zSelf; /* Name of this virtual table */
  477. char *zCostTab; /* Name of edit-cost-table */
  478. char *zVocabTab; /* Name of vocabulary table */
  479. char *zVocabWord; /* Name of vocabulary table word column */
  480. char *zVocabLang; /* Name of vocabulary table language column */
  481. amatch_rule *pRule; /* All active rules in this amatch */
  482. amatch_cost rIns; /* Generic insertion cost '' -> ? */
  483. amatch_cost rDel; /* Generic deletion cost ? -> '' */
  484. amatch_cost rSub; /* Generic substitution cost ? -> ? */
  485. sqlite3 *db; /* The database connection */
  486. sqlite3_stmt *pVCheck; /* Query to check zVocabTab */
  487. int nCursor; /* Number of active cursors */
  488. };
  489. /* A amatch cursor object */
  490. struct amatch_cursor {
  491. sqlite3_vtab_cursor base; /* Base class - must be first */
  492. sqlite3_int64 iRowid; /* The rowid of the current word */
  493. amatch_langid iLang; /* Use this language ID */
  494. amatch_cost rLimit; /* Maximum cost of any term */
  495. int nBuf; /* Space allocated for zBuf */
  496. int oomErr; /* True following an OOM error */
  497. int nWord; /* Number of amatch_word objects */
  498. char *zBuf; /* Temp-use buffer space */
  499. char *zInput; /* Input word to match against */
  500. amatch_vtab *pVtab; /* The virtual table this cursor belongs to */
  501. amatch_word *pAllWords; /* List of all amatch_word objects */
  502. amatch_word *pCurrent; /* Most recent solution */
  503. amatch_avl *pCost; /* amatch_word objects keyed by iCost */
  504. amatch_avl *pWord; /* amatch_word objects keyed by zWord */
  505. };
  506. /*
  507. ** The two input rule lists are both sorted in order of increasing
  508. ** cost. Merge them together into a single list, sorted by cost, and
  509. ** return a pointer to the head of that list.
  510. */
  511. static amatch_rule *amatchMergeRules(amatch_rule *pA, amatch_rule *pB){
  512. amatch_rule head;
  513. amatch_rule *pTail;
  514. pTail = &head;
  515. while( pA && pB ){
  516. if( pA->rCost<=pB->rCost ){
  517. pTail->pNext = pA;
  518. pTail = pA;
  519. pA = pA->pNext;
  520. }else{
  521. pTail->pNext = pB;
  522. pTail = pB;
  523. pB = pB->pNext;
  524. }
  525. }
  526. if( pA==0 ){
  527. pTail->pNext = pB;
  528. }else{
  529. pTail->pNext = pA;
  530. }
  531. return head.pNext;
  532. }
  533. /*
  534. ** Statement pStmt currently points to a row in the amatch data table. This
  535. ** function allocates and populates a amatch_rule structure according to
  536. ** the content of the row.
  537. **
  538. ** If successful, *ppRule is set to point to the new object and SQLITE_OK
  539. ** is returned. Otherwise, *ppRule is zeroed, *pzErr may be set to point
  540. ** to an error message and an SQLite error code returned.
  541. */
  542. static int amatchLoadOneRule(
  543. amatch_vtab *p, /* Fuzzer virtual table handle */
  544. sqlite3_stmt *pStmt, /* Base rule on statements current row */
  545. amatch_rule **ppRule, /* OUT: New rule object */
  546. char **pzErr /* OUT: Error message */
  547. ){
  548. sqlite3_int64 iLang = sqlite3_column_int64(pStmt, 0);
  549. const char *zFrom = (const char *)sqlite3_column_text(pStmt, 1);
  550. const char *zTo = (const char *)sqlite3_column_text(pStmt, 2);
  551. amatch_cost rCost = sqlite3_column_int(pStmt, 3);
  552. int rc = SQLITE_OK; /* Return code */
  553. int nFrom; /* Size of string zFrom, in bytes */
  554. int nTo; /* Size of string zTo, in bytes */
  555. amatch_rule *pRule = 0; /* New rule object to return */
  556. if( zFrom==0 ) zFrom = "";
  557. if( zTo==0 ) zTo = "";
  558. nFrom = (int)strlen(zFrom);
  559. nTo = (int)strlen(zTo);
  560. /* Silently ignore null transformations */
  561. if( strcmp(zFrom, zTo)==0 ){
  562. if( zFrom[0]=='?' && zFrom[1]==0 ){
  563. if( p->rSub==0 || p->rSub>rCost ) p->rSub = rCost;
  564. }
  565. *ppRule = 0;
  566. return SQLITE_OK;
  567. }
  568. if( rCost<=0 || rCost>AMATCH_MX_COST ){
  569. *pzErr = sqlite3_mprintf("%s: cost must be between 1 and %d",
  570. p->zClassName, AMATCH_MX_COST
  571. );
  572. rc = SQLITE_ERROR;
  573. }else
  574. if( nFrom>AMATCH_MX_LENGTH || nTo>AMATCH_MX_LENGTH ){
  575. *pzErr = sqlite3_mprintf("%s: maximum string length is %d",
  576. p->zClassName, AMATCH_MX_LENGTH
  577. );
  578. rc = SQLITE_ERROR;
  579. }else
  580. if( iLang<0 || iLang>AMATCH_MX_LANGID ){
  581. *pzErr = sqlite3_mprintf("%s: iLang must be between 0 and %d",
  582. p->zClassName, AMATCH_MX_LANGID
  583. );
  584. rc = SQLITE_ERROR;
  585. }else
  586. if( strcmp(zFrom,"")==0 && strcmp(zTo,"?")==0 ){
  587. if( p->rIns==0 || p->rIns>rCost ) p->rIns = rCost;
  588. }else
  589. if( strcmp(zFrom,"?")==0 && strcmp(zTo,"")==0 ){
  590. if( p->rDel==0 || p->rDel>rCost ) p->rDel = rCost;
  591. }else
  592. {
  593. pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo );
  594. if( pRule==0 ){
  595. rc = SQLITE_NOMEM;
  596. }else{
  597. memset(pRule, 0, sizeof(*pRule));
  598. pRule->zFrom = &pRule->zTo[nTo+1];
  599. pRule->nFrom = nFrom;
  600. memcpy(pRule->zFrom, zFrom, nFrom+1);
  601. memcpy(pRule->zTo, zTo, nTo+1);
  602. pRule->nTo = nTo;
  603. pRule->rCost = rCost;
  604. pRule->iLang = (int)iLang;
  605. }
  606. }
  607. *ppRule = pRule;
  608. return rc;
  609. }
  610. /*
  611. ** Free all the content in the edit-cost-table
  612. */
  613. static void amatchFreeRules(amatch_vtab *p){
  614. while( p->pRule ){
  615. amatch_rule *pRule = p->pRule;
  616. p->pRule = pRule->pNext;
  617. sqlite3_free(pRule);
  618. }
  619. p->pRule = 0;
  620. }
  621. /*
  622. ** Load the content of the amatch data table into memory.
  623. */
  624. static int amatchLoadRules(
  625. sqlite3 *db, /* Database handle */
  626. amatch_vtab *p, /* Virtual amatch table to configure */
  627. char **pzErr /* OUT: Error message */
  628. ){
  629. int rc = SQLITE_OK; /* Return code */
  630. char *zSql; /* SELECT used to read from rules table */
  631. amatch_rule *pHead = 0;
  632. zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", p->zDb, p->zCostTab);
  633. if( zSql==0 ){
  634. rc = SQLITE_NOMEM;
  635. }else{
  636. int rc2; /* finalize() return code */
  637. sqlite3_stmt *pStmt = 0;
  638. rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
  639. if( rc!=SQLITE_OK ){
  640. *pzErr = sqlite3_mprintf("%s: %s", p->zClassName, sqlite3_errmsg(db));
  641. }else if( sqlite3_column_count(pStmt)!=4 ){
  642. *pzErr = sqlite3_mprintf("%s: %s has %d columns, expected 4",
  643. p->zClassName, p->zCostTab, sqlite3_column_count(pStmt)
  644. );
  645. rc = SQLITE_ERROR;
  646. }else{
  647. while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
  648. amatch_rule *pRule = 0;
  649. rc = amatchLoadOneRule(p, pStmt, &pRule, pzErr);
  650. if( pRule ){
  651. pRule->pNext = pHead;
  652. pHead = pRule;
  653. }
  654. }
  655. }
  656. rc2 = sqlite3_finalize(pStmt);
  657. if( rc==SQLITE_OK ) rc = rc2;
  658. }
  659. sqlite3_free(zSql);
  660. /* All rules are now in a singly linked list starting at pHead. This
  661. ** block sorts them by cost and then sets amatch_vtab.pRule to point to
  662. ** point to the head of the sorted list.
  663. */
  664. if( rc==SQLITE_OK ){
  665. unsigned int i;
  666. amatch_rule *pX;
  667. amatch_rule *a[15];
  668. for(i=0; i<sizeof(a)/sizeof(a[0]); i++) a[i] = 0;
  669. while( (pX = pHead)!=0 ){
  670. pHead = pX->pNext;
  671. pX->pNext = 0;
  672. for(i=0; a[i] && i<sizeof(a)/sizeof(a[0])-1; i++){
  673. pX = amatchMergeRules(a[i], pX);
  674. a[i] = 0;
  675. }
  676. a[i] = amatchMergeRules(a[i], pX);
  677. }
  678. for(pX=a[0], i=1; i<sizeof(a)/sizeof(a[0]); i++){
  679. pX = amatchMergeRules(a[i], pX);
  680. }
  681. p->pRule = amatchMergeRules(p->pRule, pX);
  682. }else{
  683. /* An error has occurred. Setting p->pRule to point to the head of the
  684. ** allocated list ensures that the list will be cleaned up in this case.
  685. */
  686. assert( p->pRule==0 );
  687. p->pRule = pHead;
  688. }
  689. return rc;
  690. }
  691. /*
  692. ** This function converts an SQL quoted string into an unquoted string
  693. ** and returns a pointer to a buffer allocated using sqlite3_malloc()
  694. ** containing the result. The caller should eventually free this buffer
  695. ** using sqlite3_free.
  696. **
  697. ** Examples:
  698. **
  699. ** "abc" becomes abc
  700. ** 'xyz' becomes xyz
  701. ** [pqr] becomes pqr
  702. ** `mno` becomes mno
  703. */
  704. static char *amatchDequote(const char *zIn){
  705. int nIn; /* Size of input string, in bytes */
  706. char *zOut; /* Output (dequoted) string */
  707. nIn = (int)strlen(zIn);
  708. zOut = sqlite3_malloc(nIn+1);
  709. if( zOut ){
  710. char q = zIn[0]; /* Quote character (if any ) */
  711. if( q!='[' && q!= '\'' && q!='"' && q!='`' ){
  712. memcpy(zOut, zIn, nIn+1);
  713. }else{
  714. int iOut = 0; /* Index of next byte to write to output */
  715. int iIn; /* Index of next byte to read from input */
  716. if( q=='[' ) q = ']';
  717. for(iIn=1; iIn<nIn; iIn++){
  718. if( zIn[iIn]==q ) iIn++;
  719. zOut[iOut++] = zIn[iIn];
  720. }
  721. }
  722. assert( (int)strlen(zOut)<=nIn );
  723. }
  724. return zOut;
  725. }
  726. /*
  727. ** Deallocate the pVCheck prepared statement.
  728. */
  729. static void amatchVCheckClear(amatch_vtab *p){
  730. if( p->pVCheck ){
  731. sqlite3_finalize(p->pVCheck);
  732. p->pVCheck = 0;
  733. }
  734. }
  735. /*
  736. ** Deallocate an amatch_vtab object
  737. */
  738. static void amatchFree(amatch_vtab *p){
  739. if( p ){
  740. amatchFreeRules(p);
  741. amatchVCheckClear(p);
  742. sqlite3_free(p->zClassName);
  743. sqlite3_free(p->zDb);
  744. sqlite3_free(p->zCostTab);
  745. sqlite3_free(p->zVocabTab);
  746. sqlite3_free(p->zVocabWord);
  747. sqlite3_free(p->zVocabLang);
  748. sqlite3_free(p->zSelf);
  749. memset(p, 0, sizeof(*p));
  750. sqlite3_free(p);
  751. }
  752. }
  753. /*
  754. ** xDisconnect/xDestroy method for the amatch module.
  755. */
  756. static int amatchDisconnect(sqlite3_vtab *pVtab){
  757. amatch_vtab *p = (amatch_vtab*)pVtab;
  758. assert( p->nCursor==0 );
  759. amatchFree(p);
  760. return SQLITE_OK;
  761. }
  762. /*
  763. ** Check to see if the argument is of the form:
  764. **
  765. ** KEY = VALUE
  766. **
  767. ** If it is, return a pointer to the first character of VALUE.
  768. ** If not, return NULL. Spaces around the = are ignored.
  769. */
  770. static const char *amatchValueOfKey(const char *zKey, const char *zStr){
  771. int nKey = (int)strlen(zKey);
  772. int nStr = (int)strlen(zStr);
  773. int i;
  774. if( nStr<nKey+1 ) return 0;
  775. if( memcmp(zStr, zKey, nKey)!=0 ) return 0;
  776. for(i=nKey; isspace(zStr[i]); i++){}
  777. if( zStr[i]!='=' ) return 0;
  778. i++;
  779. while( isspace(zStr[i]) ){ i++; }
  780. return zStr+i;
  781. }
  782. /*
  783. ** xConnect/xCreate method for the amatch module. Arguments are:
  784. **
  785. ** argv[0] -> module name ("approximate_match")
  786. ** argv[1] -> database name
  787. ** argv[2] -> table name
  788. ** argv[3...] -> arguments
  789. */
  790. static int amatchConnect(
  791. sqlite3 *db,
  792. void *pAux,
  793. int argc, const char *const*argv,
  794. sqlite3_vtab **ppVtab,
  795. char **pzErr
  796. ){
  797. int rc = SQLITE_OK; /* Return code */
  798. amatch_vtab *pNew = 0; /* New virtual table */
  799. const char *zModule = argv[0];
  800. const char *zDb = argv[1];
  801. const char *zVal;
  802. int i;
  803. (void)pAux;
  804. *ppVtab = 0;
  805. pNew = sqlite3_malloc( sizeof(*pNew) );
  806. if( pNew==0 ) return SQLITE_NOMEM;
  807. rc = SQLITE_NOMEM;
  808. memset(pNew, 0, sizeof(*pNew));
  809. pNew->db = db;
  810. pNew->zClassName = sqlite3_mprintf("%s", zModule);
  811. if( pNew->zClassName==0 ) goto amatchConnectError;
  812. pNew->zDb = sqlite3_mprintf("%s", zDb);
  813. if( pNew->zDb==0 ) goto amatchConnectError;
  814. pNew->zSelf = sqlite3_mprintf("%s", argv[2]);
  815. if( pNew->zSelf==0 ) goto amatchConnectError;
  816. for(i=3; i<argc; i++){
  817. zVal = amatchValueOfKey("vocabulary_table", argv[i]);
  818. if( zVal ){
  819. sqlite3_free(pNew->zVocabTab);
  820. pNew->zVocabTab = amatchDequote(zVal);
  821. if( pNew->zVocabTab==0 ) goto amatchConnectError;
  822. continue;
  823. }
  824. zVal = amatchValueOfKey("vocabulary_word", argv[i]);
  825. if( zVal ){
  826. sqlite3_free(pNew->zVocabWord);
  827. pNew->zVocabWord = amatchDequote(zVal);
  828. if( pNew->zVocabWord==0 ) goto amatchConnectError;
  829. continue;
  830. }
  831. zVal = amatchValueOfKey("vocabulary_language", argv[i]);
  832. if( zVal ){
  833. sqlite3_free(pNew->zVocabLang);
  834. pNew->zVocabLang = amatchDequote(zVal);
  835. if( pNew->zVocabLang==0 ) goto amatchConnectError;
  836. continue;
  837. }
  838. zVal = amatchValueOfKey("edit_distances", argv[i]);
  839. if( zVal ){
  840. sqlite3_free(pNew->zCostTab);
  841. pNew->zCostTab = amatchDequote(zVal);
  842. if( pNew->zCostTab==0 ) goto amatchConnectError;
  843. continue;
  844. }
  845. *pzErr = sqlite3_mprintf("unrecognized argument: [%s]\n", argv[i]);
  846. amatchFree(pNew);
  847. *ppVtab = 0;
  848. return SQLITE_ERROR;
  849. }
  850. rc = SQLITE_OK;
  851. if( pNew->zCostTab==0 ){
  852. *pzErr = sqlite3_mprintf("no edit_distances table specified");
  853. rc = SQLITE_ERROR;
  854. }else{
  855. rc = amatchLoadRules(db, pNew, pzErr);
  856. }
  857. if( rc==SQLITE_OK ){
  858. rc = sqlite3_declare_vtab(db,
  859. "CREATE TABLE x(word,distance,language,"
  860. "command HIDDEN,nword HIDDEN)"
  861. );
  862. #define AMATCH_COL_WORD 0
  863. #define AMATCH_COL_DISTANCE 1
  864. #define AMATCH_COL_LANGUAGE 2
  865. #define AMATCH_COL_COMMAND 3
  866. #define AMATCH_COL_NWORD 4
  867. }
  868. if( rc!=SQLITE_OK ){
  869. amatchFree(pNew);
  870. }
  871. *ppVtab = &pNew->base;
  872. return rc;
  873. amatchConnectError:
  874. amatchFree(pNew);
  875. return rc;
  876. }
  877. /*
  878. ** Open a new amatch cursor.
  879. */
  880. static int amatchOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
  881. amatch_vtab *p = (amatch_vtab*)pVTab;
  882. amatch_cursor *pCur;
  883. pCur = sqlite3_malloc( sizeof(*pCur) );
  884. if( pCur==0 ) return SQLITE_NOMEM;
  885. memset(pCur, 0, sizeof(*pCur));
  886. pCur->pVtab = p;
  887. *ppCursor = &pCur->base;
  888. p->nCursor++;
  889. return SQLITE_OK;
  890. }
  891. /*
  892. ** Free up all the memory allocated by a cursor. Set it rLimit to 0
  893. ** to indicate that it is at EOF.
  894. */
  895. static void amatchClearCursor(amatch_cursor *pCur){
  896. amatch_word *pWord, *pNextWord;
  897. for(pWord=pCur->pAllWords; pWord; pWord=pNextWord){
  898. pNextWord = pWord->pNext;
  899. sqlite3_free(pWord);
  900. }
  901. pCur->pAllWords = 0;
  902. sqlite3_free(pCur->zInput);
  903. pCur->zInput = 0;
  904. sqlite3_free(pCur->zBuf);
  905. pCur->zBuf = 0;
  906. pCur->nBuf = 0;
  907. pCur->pCost = 0;
  908. pCur->pWord = 0;
  909. pCur->pCurrent = 0;
  910. pCur->rLimit = 1000000;
  911. pCur->iLang = 0;
  912. pCur->nWord = 0;
  913. }
  914. /*
  915. ** Close a amatch cursor.
  916. */
  917. static int amatchClose(sqlite3_vtab_cursor *cur){
  918. amatch_cursor *pCur = (amatch_cursor *)cur;
  919. amatchClearCursor(pCur);
  920. pCur->pVtab->nCursor--;
  921. sqlite3_free(pCur);
  922. return SQLITE_OK;
  923. }
  924. /*
  925. ** Render a 24-bit unsigned integer as a 4-byte base-64 number.
  926. */
  927. static void amatchEncodeInt(int x, char *z){
  928. static const char a[] =
  929. "0123456789"
  930. "ABCDEFGHIJ"
  931. "KLMNOPQRST"
  932. "UVWXYZ^abc"
  933. "defghijklm"
  934. "nopqrstuvw"
  935. "xyz~";
  936. z[0] = a[(x>>18)&0x3f];
  937. z[1] = a[(x>>12)&0x3f];
  938. z[2] = a[(x>>6)&0x3f];
  939. z[3] = a[x&0x3f];
  940. }
  941. /*
  942. ** Write the zCost[] field for a amatch_word object
  943. */
  944. static void amatchWriteCost(amatch_word *pWord){
  945. amatchEncodeInt(pWord->rCost, pWord->zCost);
  946. amatchEncodeInt(pWord->iSeq, pWord->zCost+4);
  947. pWord->zCost[8] = 0;
  948. }
  949. /*
  950. ** Add a new amatch_word object to the queue.
  951. **
  952. ** If a prior amatch_word object with the same zWord, and nMatch
  953. ** already exists, update its rCost (if the new rCost is less) but
  954. ** otherwise leave it unchanged. Do not add a duplicate.
  955. **
  956. ** Do nothing if the cost exceeds threshold.
  957. */
  958. static void amatchAddWord(
  959. amatch_cursor *pCur,
  960. amatch_cost rCost,
  961. int nMatch,
  962. const char *zWordBase,
  963. const char *zWordTail
  964. ){
  965. amatch_word *pWord;
  966. amatch_avl *pNode;
  967. amatch_avl *pOther;
  968. int nBase, nTail;
  969. char zBuf[4];
  970. if( rCost>pCur->rLimit ){
  971. return;
  972. }
  973. nBase = (int)strlen(zWordBase);
  974. nTail = (int)strlen(zWordTail);
  975. if( nBase+nTail+3>pCur->nBuf ){
  976. pCur->nBuf = nBase+nTail+100;
  977. pCur->zBuf = sqlite3_realloc(pCur->zBuf, pCur->nBuf);
  978. if( pCur->zBuf==0 ){
  979. pCur->nBuf = 0;
  980. return;
  981. }
  982. }
  983. amatchEncodeInt(nMatch, zBuf);
  984. memcpy(pCur->zBuf, zBuf+2, 2);
  985. memcpy(pCur->zBuf+2, zWordBase, nBase);
  986. memcpy(pCur->zBuf+2+nBase, zWordTail, nTail+1);
  987. pNode = amatchAvlSearch(pCur->pWord, pCur->zBuf);
  988. if( pNode ){
  989. pWord = pNode->pWord;
  990. if( pWord->rCost>rCost ){
  991. #ifdef AMATCH_TRACE_1
  992. printf("UPDATE [%s][%.*s^%s] %d (\"%s\" \"%s\")\n",
  993. pWord->zWord+2, pWord->nMatch, pCur->zInput, pCur->zInput,
  994. pWord->rCost, pWord->zWord, pWord->zCost);
  995. #endif
  996. amatchAvlRemove(&pCur->pCost, &pWord->sCost);
  997. pWord->rCost = rCost;
  998. amatchWriteCost(pWord);
  999. #ifdef AMATCH_TRACE_1
  1000. printf(" ---> %d (\"%s\" \"%s\")\n",
  1001. pWord->rCost, pWord->zWord, pWord->zCost);
  1002. #endif
  1003. pOther = amatchAvlInsert(&pCur->pCost, &pWord->sCost);
  1004. assert( pOther==0 ); (void)pOther;
  1005. }
  1006. return;
  1007. }
  1008. pWord = sqlite3_malloc( sizeof(*pWord) + nBase + nTail - 1 );
  1009. if( pWord==0 ) return;
  1010. memset(pWord, 0, sizeof(*pWord));
  1011. pWord->rCost = rCost;
  1012. pWord->iSeq = pCur->nWord++;
  1013. amatchWriteCost(pWord);
  1014. pWord->nMatch = nMatch;
  1015. pWord->pNext = pCur->pAllWords;
  1016. pCur->pAllWords = pWord;
  1017. pWord->sCost.zKey = pWord->zCost;
  1018. pWord->sCost.pWord = pWord;
  1019. pOther = amatchAvlInsert(&pCur->pCost, &pWord->sCost);
  1020. assert( pOther==0 ); (void)pOther;
  1021. pWord->sWord.zKey = pWord->zWord;
  1022. pWord->sWord.pWord = pWord;
  1023. strcpy(pWord->zWord, pCur->zBuf);
  1024. pOther = amatchAvlInsert(&pCur->pWord, &pWord->sWord);
  1025. assert( pOther==0 ); (void)pOther;
  1026. #ifdef AMATCH_TRACE_1
  1027. printf("INSERT [%s][%.*s^%s] %d (\"%s\" \"%s\")\n", pWord->zWord+2,
  1028. pWord->nMatch, pCur->zInput, pCur->zInput+pWord->nMatch, rCost,
  1029. pWord->zWord, pWord->zCost);
  1030. #endif
  1031. }
  1032. /*
  1033. ** Advance a cursor to its next row of output
  1034. */
  1035. static int amatchNext(sqlite3_vtab_cursor *cur){
  1036. amatch_cursor *pCur = (amatch_cursor*)cur;
  1037. amatch_word *pWord = 0;
  1038. amatch_avl *pNode;
  1039. int isMatch = 0;
  1040. amatch_vtab *p = pCur->pVtab;
  1041. int nWord;
  1042. int rc;
  1043. int i;
  1044. const char *zW;
  1045. amatch_rule *pRule;
  1046. char *zBuf = 0;
  1047. char nBuf = 0;
  1048. char zNext[8];
  1049. char zNextIn[8];
  1050. int nNextIn;
  1051. if( p->pVCheck==0 ){
  1052. char *zSql;
  1053. if( p->zVocabLang && p->zVocabLang[0] ){
  1054. zSql = sqlite3_mprintf(
  1055. "SELECT \"%w\" FROM \"%w\"",
  1056. " WHERE \"%w\">=?1 AND \"%w\"=?2"
  1057. " ORDER BY 1",
  1058. p->zVocabWord, p->zVocabTab,
  1059. p->zVocabWord, p->zVocabLang
  1060. );
  1061. }else{
  1062. zSql = sqlite3_mprintf(
  1063. "SELECT \"%w\" FROM \"%w\""
  1064. " WHERE \"%w\">=?1"
  1065. " ORDER BY 1",
  1066. p->zVocabWord, p->zVocabTab,
  1067. p->zVocabWord
  1068. );
  1069. }
  1070. rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->pVCheck, 0);
  1071. sqlite3_free(zSql);
  1072. if( rc ) return rc;
  1073. }
  1074. sqlite3_bind_int(p->pVCheck, 2, pCur->iLang);
  1075. do{
  1076. pNode = amatchAvlFirst(pCur->pCost);
  1077. if( pNode==0 ){
  1078. pWord = 0;
  1079. break;
  1080. }
  1081. pWord = pNode->pWord;
  1082. amatchAvlRemove(&pCur->pCost, &pWord->sCost);
  1083. #ifdef AMATCH_TRACE_1
  1084. printf("PROCESS [%s][%.*s^%s] %d (\"%s\" \"%s\")\n",
  1085. pWord->zWord+2, pWord->nMatch, pCur->zInput, pCur->zInput+pWord->nMatch,
  1086. pWord->rCost, pWord->zWord, pWord->zCost);
  1087. #endif
  1088. nWord = (int)strlen(pWord->zWord+2);
  1089. if( nWord+20>nBuf ){
  1090. nBuf = nWord+100;
  1091. zBuf = sqlite3_realloc(zBuf, nBuf);
  1092. if( zBuf==0 ) return SQLITE_NOMEM;
  1093. }
  1094. strcpy(zBuf, pWord->zWord+2);
  1095. zNext[0] = 0;
  1096. zNextIn[0] = pCur->zInput[pWord->nMatch];
  1097. if( zNextIn[0] ){
  1098. for(i=1; i<=4 && (pCur->zInput[pWord->nMatch+i]&0xc0)==0x80; i++){
  1099. zNextIn[i] = pCur->zInput[pWord->nMatch+i];
  1100. }
  1101. zNextIn[i] = 0;
  1102. nNextIn = i;
  1103. }else{
  1104. nNextIn = 0;
  1105. }
  1106. if( zNextIn[0] && zNextIn[0]!='*' ){
  1107. sqlite3_reset(p->pVCheck);
  1108. strcat(zBuf, zNextIn);
  1109. sqlite3_bind_text(p->pVCheck, 1, zBuf, nWord+nNextIn, SQLITE_STATIC);
  1110. rc = sqlite3_step(p->pVCheck);
  1111. if( rc==SQLITE_ROW ){
  1112. zW = (const char*)sqlite3_column_text(p->pVCheck, 0);
  1113. if( strncmp(zBuf, zW, nWord+nNextIn)==0 ){
  1114. amatchAddWord(pCur, pWord->rCost, pWord->nMatch+nNextIn, zBuf, "");
  1115. }
  1116. }
  1117. zBuf[nWord] = 0;
  1118. }
  1119. while( 1 ){
  1120. strcpy(zBuf+nWord, zNext);
  1121. sqlite3_reset(p->pVCheck);
  1122. sqlite3_bind_text(p->pVCheck, 1, zBuf, -1, SQLITE_TRANSIENT);
  1123. rc = sqlite3_step(p->pVCheck);
  1124. if( rc!=SQLITE_ROW ) break;
  1125. zW = (const char*)sqlite3_column_text(p->pVCheck, 0);
  1126. strcpy(zBuf+nWord, zNext);
  1127. if( strncmp(zW, zBuf, nWord)!=0 ) break;
  1128. if( (zNextIn[0]=='*' && zNextIn[1]==0)
  1129. || (zNextIn[0]==0 && zW[nWord]==0)
  1130. ){
  1131. isMatch = 1;
  1132. zNextIn[0] = 0;
  1133. nNextIn = 0;
  1134. break;
  1135. }
  1136. zNext[0] = zW[nWord];
  1137. for(i=1; i<=4 && (zW[nWord+i]&0xc0)==0x80; i++){
  1138. zNext[i] = zW[nWord+i];
  1139. }
  1140. zNext[i] = 0;
  1141. zBuf[nWord] = 0;
  1142. if( p->rIns>0 ){
  1143. amatchAddWord(pCur, pWord->rCost+p->rIns, pWord->nMatch,
  1144. zBuf, zNext);
  1145. }
  1146. if( p->rSub>0 ){
  1147. amatchAddWord(pCur, pWord->rCost+p->rSub, pWord->nMatch+nNextIn,
  1148. zBuf, zNext);
  1149. }
  1150. if( p->rIns<0 && p->rSub<0 ) break;
  1151. zNext[i-1]++; /* FIX ME */
  1152. }
  1153. sqlite3_reset(p->pVCheck);
  1154. if( p->rDel>0 ){
  1155. zBuf[nWord] = 0;
  1156. amatchAddWord(pCur, pWord->rCost+p->rDel, pWord->nMatch+nNextIn,
  1157. zBuf, "");
  1158. }
  1159. for(pRule=p->pRule; pRule; pRule=pRule->pNext){
  1160. if( pRule->iLang!=pCur->iLang ) continue;
  1161. if( strncmp(pRule->zFrom, pCur->zInput+pWord->nMatch, pRule->nFrom)==0 ){
  1162. amatchAddWord(pCur, pWord->rCost+pRule->rCost,
  1163. pWord->nMatch+pRule->nFrom, pWord->zWord+2, pRule->zTo);
  1164. }
  1165. }
  1166. }while( !isMatch );
  1167. pCur->pCurrent = pWord;
  1168. sqlite3_free(zBuf);
  1169. return SQLITE_OK;
  1170. }
  1171. /*
  1172. ** Called to "rewind" a cursor back to the beginning so that
  1173. ** it starts its output over again. Always called at least once
  1174. ** prior to any amatchColumn, amatchRowid, or amatchEof call.
  1175. */
  1176. static int amatchFilter(
  1177. sqlite3_vtab_cursor *pVtabCursor,
  1178. int idxNum, const char *idxStr,
  1179. int argc, sqlite3_value **argv
  1180. ){
  1181. amatch_cursor *pCur = (amatch_cursor *)pVtabCursor;
  1182. const char *zWord = "*";
  1183. int idx;
  1184. amatchClearCursor(pCur);
  1185. idx = 0;
  1186. if( idxNum & 1 ){
  1187. zWord = (const char*)sqlite3_value_text(argv[0]);
  1188. idx++;
  1189. }
  1190. if( idxNum & 2 ){
  1191. pCur->rLimit = (amatch_cost)sqlite3_value_int(argv[idx]);
  1192. idx++;
  1193. }
  1194. if( idxNum & 4 ){
  1195. pCur->iLang = (amatch_cost)sqlite3_value_int(argv[idx]);
  1196. idx++;
  1197. }
  1198. pCur->zInput = sqlite3_mprintf("%s", zWord);
  1199. if( pCur->zInput==0 ) return SQLITE_NOMEM;
  1200. amatchAddWord(pCur, 0, 0, "", "");
  1201. amatchNext(pVtabCursor);
  1202. return SQLITE_OK;
  1203. }
  1204. /*
  1205. ** Only the word and distance columns have values. All other columns
  1206. ** return NULL
  1207. */
  1208. static int amatchColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
  1209. amatch_cursor *pCur = (amatch_cursor*)cur;
  1210. switch( i ){
  1211. case AMATCH_COL_WORD: {
  1212. sqlite3_result_text(ctx, pCur->pCurrent->zWord+2, -1, SQLITE_STATIC);
  1213. break;
  1214. }
  1215. case AMATCH_COL_DISTANCE: {
  1216. sqlite3_result_int(ctx, pCur->pCurrent->rCost);
  1217. break;
  1218. }
  1219. case AMATCH_COL_LANGUAGE: {
  1220. sqlite3_result_int(ctx, pCur->iLang);
  1221. break;
  1222. }
  1223. case AMATCH_COL_NWORD: {
  1224. sqlite3_result_int(ctx, pCur->nWord);
  1225. break;
  1226. }
  1227. default: {
  1228. sqlite3_result_null(ctx);
  1229. break;
  1230. }
  1231. }
  1232. return SQLITE_OK;
  1233. }
  1234. /*
  1235. ** The rowid.
  1236. */
  1237. static int amatchRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
  1238. amatch_cursor *pCur = (amatch_cursor*)cur;
  1239. *pRowid = pCur->iRowid;
  1240. return SQLITE_OK;
  1241. }
  1242. /*
  1243. ** EOF indicator
  1244. */
  1245. static int amatchEof(sqlite3_vtab_cursor *cur){
  1246. amatch_cursor *pCur = (amatch_cursor*)cur;
  1247. return pCur->pCurrent==0;
  1248. }
  1249. /*
  1250. ** Search for terms of these forms:
  1251. **
  1252. ** (A) word MATCH $str
  1253. ** (B1) distance < $value
  1254. ** (B2) distance <= $value
  1255. ** (C) language == $language
  1256. **
  1257. ** The distance< and distance<= are both treated as distance<=.
  1258. ** The query plan number is a bit vector:
  1259. **
  1260. ** bit 1: Term of the form (A) found
  1261. ** bit 2: Term like (B1) or (B2) found
  1262. ** bit 3: Term like (C) found
  1263. **
  1264. ** If bit-1 is set, $str is always in filter.argv[0]. If bit-2 is set
  1265. ** then $value is in filter.argv[0] if bit-1 is clear and is in
  1266. ** filter.argv[1] if bit-1 is set. If bit-3 is set, then $ruleid is
  1267. ** in filter.argv[0] if bit-1 and bit-2 are both zero, is in
  1268. ** filter.argv[1] if exactly one of bit-1 and bit-2 are set, and is in
  1269. ** filter.argv[2] if both bit-1 and bit-2 are set.
  1270. */
  1271. static int amatchBestIndex(
  1272. sqlite3_vtab *tab,
  1273. sqlite3_index_info *pIdxInfo
  1274. ){
  1275. int iPlan = 0;
  1276. int iDistTerm = -1;
  1277. int iLangTerm = -1;
  1278. int i;
  1279. const struct sqlite3_index_constraint *pConstraint;
  1280. (void)tab;
  1281. pConstraint = pIdxInfo->aConstraint;
  1282. for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
  1283. if( pConstraint->usable==0 ) continue;
  1284. if( (iPlan & 1)==0
  1285. && pConstraint->iColumn==0
  1286. && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH
  1287. ){
  1288. iPlan |= 1;
  1289. pIdxInfo->aConstraintUsage[i].argvIndex = 1;
  1290. pIdxInfo->aConstraintUsage[i].omit = 1;
  1291. }
  1292. if( (iPlan & 2)==0
  1293. && pConstraint->iColumn==1
  1294. && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT
  1295. || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE)
  1296. ){
  1297. iPlan |= 2;
  1298. iDistTerm = i;
  1299. }
  1300. if( (iPlan & 4)==0
  1301. && pConstraint->iColumn==2
  1302. && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ
  1303. ){
  1304. iPlan |= 4;
  1305. pIdxInfo->aConstraintUsage[i].omit = 1;
  1306. iLangTerm = i;
  1307. }
  1308. }
  1309. if( iPlan & 2 ){
  1310. pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1+((iPlan&1)!=0);
  1311. }
  1312. if( iPlan & 4 ){
  1313. int idx = 1;
  1314. if( iPlan & 1 ) idx++;
  1315. if( iPlan & 2 ) idx++;
  1316. pIdxInfo->aConstraintUsage[iLangTerm].argvIndex = idx;
  1317. }
  1318. pIdxInfo->idxNum = iPlan;
  1319. if( pIdxInfo->nOrderBy==1
  1320. && pIdxInfo->aOrderBy[0].iColumn==1
  1321. && pIdxInfo->aOrderBy[0].desc==0
  1322. ){
  1323. pIdxInfo->orderByConsumed = 1;
  1324. }
  1325. pIdxInfo->estimatedCost = (double)10000;
  1326. return SQLITE_OK;
  1327. }
  1328. /*
  1329. ** The xUpdate() method.
  1330. **
  1331. ** This implementation disallows DELETE and UPDATE. The only thing
  1332. ** allowed is INSERT into the "command" column.
  1333. */
  1334. static int amatchUpdate(
  1335. sqlite3_vtab *pVTab,
  1336. int argc,
  1337. sqlite3_value **argv,
  1338. sqlite_int64 *pRowid
  1339. ){
  1340. amatch_vtab *p = (amatch_vtab*)pVTab;
  1341. const unsigned char *zCmd;
  1342. (void)pRowid;
  1343. if( argc==1 ){
  1344. pVTab->zErrMsg = sqlite3_mprintf("DELETE from %s is not allowed",
  1345. p->zSelf);
  1346. return SQLITE_ERROR;
  1347. }
  1348. if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){
  1349. pVTab->zErrMsg = sqlite3_mprintf("UPDATE of %s is not allowed",
  1350. p->zSelf);
  1351. return SQLITE_ERROR;
  1352. }
  1353. if( sqlite3_value_type(argv[2+AMATCH_COL_WORD])!=SQLITE_NULL
  1354. || sqlite3_value_type(argv[2+AMATCH_COL_DISTANCE])!=SQLITE_NULL
  1355. || sqlite3_value_type(argv[2+AMATCH_COL_LANGUAGE])!=SQLITE_NULL
  1356. ){
  1357. pVTab->zErrMsg = sqlite3_mprintf(
  1358. "INSERT INTO %s allowed for column [command] only", p->zSelf);
  1359. return SQLITE_ERROR;
  1360. }
  1361. zCmd = sqlite3_value_text(argv[2+AMATCH_COL_COMMAND]);
  1362. if( zCmd==0 ) return SQLITE_OK;
  1363. return SQLITE_OK;
  1364. }
  1365. /*
  1366. ** A virtual table module that implements the "approximate_match".
  1367. */
  1368. static sqlite3_module amatchModule = {
  1369. 0, /* iVersion */
  1370. amatchConnect, /* xCreate */
  1371. amatchConnect, /* xConnect */
  1372. amatchBestIndex, /* xBestIndex */
  1373. amatchDisconnect, /* xDisconnect */
  1374. amatchDisconnect, /* xDestroy */
  1375. amatchOpen, /* xOpen - open a cursor */
  1376. amatchClose, /* xClose - close a cursor */
  1377. amatchFilter, /* xFilter - configure scan constraints */
  1378. amatchNext, /* xNext - advance a cursor */
  1379. amatchEof, /* xEof - check for end of scan */
  1380. amatchColumn, /* xColumn - read data */
  1381. amatchRowid, /* xRowid - read data */
  1382. amatchUpdate, /* xUpdate */
  1383. 0, /* xBegin */
  1384. 0, /* xSync */
  1385. 0, /* xCommit */
  1386. 0, /* xRollback */
  1387. 0, /* xFindMethod */
  1388. 0, /* xRename */
  1389. 0, /* xSavepoint */
  1390. 0, /* xRelease */
  1391. 0 /* xRollbackTo */
  1392. };
  1393. #endif /* SQLITE_OMIT_VIRTUALTABLE */
  1394. /*
  1395. ** Register the amatch virtual table
  1396. */
  1397. #ifdef _WIN32
  1398. __declspec(dllexport)
  1399. #endif
  1400. int sqlite3_amatch_init(
  1401. sqlite3 *db,
  1402. char **pzErrMsg,
  1403. const sqlite3_api_routines *pApi
  1404. ){
  1405. int rc = SQLITE_OK;
  1406. SQLITE_EXTENSION_INIT2(pApi);
  1407. (void)pzErrMsg; /* Not used */
  1408. #ifndef SQLITE_OMIT_VIRTUALTABLE
  1409. rc = sqlite3_create_module(db, "approximate_match", &amatchModule, 0);
  1410. #endif /* SQLITE_OMIT_VIRTUALTABLE */
  1411. return rc;
  1412. }