From: dan <dan@noemail.net>
Date: Sat, 10 Jan 2015 20:34:27 +0000 (+0000)
Subject: Fix some documentation issues in fts5.
X-Git-Tag: version-3.8.11~114^2~108
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=27277c4e3c0fa0a05ae87b7b6be6a8380c52ecaf;p=thirdparty%2Fsqlite.git

Fix some documentation issues in fts5.

FossilOrigin-Name: 512e1bdb4093b59d1494dfc63391476eadd52aea
---

diff --git a/ext/fts5/extract_api_docs.tcl b/ext/fts5/extract_api_docs.tcl
index 81fe4cde53..27f136a99b 100644
--- a/ext/fts5/extract_api_docs.tcl
+++ b/ext/fts5/extract_api_docs.tcl
@@ -17,14 +17,17 @@
 
 set ::fts5_docs_output ""
 if {[info commands hd_putsnl]==""} {
+  if {[llength $argv]>0} { set ::extract_api_docs_mode [lindex $argv 0] }
   proc output {text} {
     puts $text
   }
 } else {
   proc output {text} {
-    append ::fts5_docs_output $text
+    append ::fts5_docs_output "$text\n"
   }
 }
+if {[info exists ::extract_api_docs_mode]==0} {set ::extract_api_docs_mode api}
+
 
 set input_file [file join [file dir [info script]] fts5.h]
 set fd [open $input_file]
@@ -41,7 +44,7 @@ close $fd
 proc get_struct_members {data} {
 
   # Extract the structure definition from the fts5.h file.
-  regexp "struct Fts5ExtensionApi {(.*)};" $data -> defn
+  regexp "struct Fts5ExtensionApi {(.*?)};" $data -> defn
 
   # Remove all comments from the structure definition
   regsub -all {/[*].*?[*]/} $defn {} defn2
@@ -95,44 +98,136 @@ proc get_struct_docs {data names} {
   set res
 }
 
-# Initialize global array M as a map from Fts5StructureApi member name
-# to member definition. i.e.
-#
-#   iVersion  -> {int iVersion}
-#   xUserData -> {void *(*xUserData)(Fts5Context*)}
-#   ...
-#
-array set M [get_struct_members $data]
+proc get_tokenizer_docs {data} {
+  regexp {(xCreate:.*?)[*]/} $data -> docs
 
-# Initialize global list D as a map from section name to documentation
-# text. Most (all?) section names are structure member names.
-#
-set D [get_struct_docs $data [array names M]]
+  set res "<dl>\n"
+  foreach line [split [string trim $docs] "\n"] {
+    regexp {[*][*](.*)} $line -> line
+    if {[regexp {^ ?x.*:} $line]} {
+      append res "<dt><b>$line</b></dt><dd><p style=margin-top:0>\n"
+      continue
+    }
+    if {[string trim $line] == ""} {
+      append res "<p>\n"
+    } else {
+      append res "$line\n"
+    }
+  }
+  append res "</dl>\n"
+
+  set res
+}
+
+proc get_api_docs {data} {
+  # Initialize global array M as a map from Fts5StructureApi member name
+  # to member definition. i.e.
+  #
+  #   iVersion  -> {int iVersion}
+  #   xUserData -> {void *(*xUserData)(Fts5Context*)}
+  #   ...
+  #
+  array set M [get_struct_members $data]
+  
+  # Initialize global list D as a map from section name to documentation
+  # text. Most (all?) section names are structure member names.
+  #
+  set D [get_struct_docs $data [array names M]]
+  
+  foreach {sub docs} $D {
+    if {[info exists M($sub)]} {
+      set hdr $M($sub)
+      set link " id=$sub"
+    } else {
+      set link ""
+    }
 
-foreach {hdr docs} $D {
-  if {[info exists M($hdr)]} {
-    set hdr $M($hdr)
+    output "<hr color=#eeeee style=\"margin:1em 8.4ex 0 8.4ex;\"$link>"
+    set style "padding-left:6ex;font-size:1.4em;display:block"
+    output "<h style=\"$style\"><pre>$hdr</pre></h>"
+  
+    set mode ""
+    set bEmpty 1
+    foreach line [split [string trim $docs] "\n"] {
+      if {[string trim $line]==""} {
+        if {$mode != ""} {output "</$mode>"}
+        set mode ""
+      } elseif {$mode == ""} {
+        if {[regexp {^     } $line]} {
+          set mode codeblock
+        } else {
+          set mode p
+        }
+        output "<$mode>"
+      }
+      output $line
+    }
+    if {$mode != ""} {output "</$mode>"}
   }
-  output "<h style=\"font-size:1.4em;background-color:#EEEEEE;display:block\"><pre>  $hdr</pre></h>"
+}
 
-  set mode ""
-  set bEmpty 1
-  foreach line [split [string trim $docs] "\n"] {
-    if {[string trim $line]==""} {
-      if {$mode != ""} {output "</$mode>"}
-      set mode ""
-    } elseif {$mode == ""} {
-      if {[regexp {^     } $line]} {
-        set mode codeblock
-      } else {
-        set mode p
+proc get_fts5_struct {data start end} {
+  set res ""
+  set bOut 0
+  foreach line [split $data "\n"] {
+    if {$bOut==0} {
+      if {[regexp $start $line]} {
+        set bOut 1
       }
-      output "<$mode>"
     }
-    output $line
+
+    if {$bOut} {
+      append res "$line\n"
+    }
+
+    if {$bOut} {
+      if {[regexp $end $line]} {
+        set bOut 0
+      }
+    }
+  }
+
+  set map [list /* <i>/* */ */</i>]
+  string map $map $res
+}
+
+proc main {data} {
+  switch $::extract_api_docs_mode {
+    fts5_api {
+      output [get_fts5_struct $data "typedef struct fts5_api" "^\};"]
+    }
+
+    fts5_tokenizer {
+      output [get_fts5_struct $data "typedef struct Fts5Tokenizer" "^\};"]
+    }
+
+    fts5_extension {
+      output [get_fts5_struct $data "typedef.*Fts5ExtensionApi" "^.;"]
+    }
+
+    Fts5ExtensionApi {
+      set struct [get_fts5_struct $data "^struct Fts5ExtensionApi" "^.;"]
+      set map [list]
+      foreach {k v} [get_struct_members $data] {
+        if {[string match x* $k]==0} continue
+        lappend map $k "<a href=#$k>$k</a>"
+      }
+      output [string map $map $struct]
+    }
+
+    api {
+      get_api_docs $data
+    }
+
+    tokenizer_api {
+      output [get_tokenizer_docs $data]
+    }
+
+    default {
+    }
   }
-  if {$mode != ""} {output "</$mode>"}
 }
+main $data
 
 set ::fts5_docs_output
 
diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h
index 1b9160f05b..8e244f3992 100644
--- a/ext/fts5/fts5.h
+++ b/ext/fts5/fts5.h
@@ -100,7 +100,7 @@ typedef void (*fts5_extension_function)(
 **   This API function is used to query the FTS table for phrase iPhrase
 **   of the current query. Specifically, a query equivalent to:
 **
-**       ... FROM ftstable WHERE ftstable MATCH $p ORDER BY DESC
+**       ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid DESC
 **
 **   with $p set to a phrase equivalent to the phrase iPhrase of the
 **   current query is executed. For each row visited, the callback function
@@ -212,10 +212,11 @@ struct Fts5ExtensionApi {
 **
 **   The first argument passed to this function is a copy of the (void*)
 **   pointer provided by the application when the fts5_tokenizer object
-**   was registered with SQLite. The second and third arguments are an
-**   array of nul-terminated strings containing the tokenizer arguments,
-**   if any, specified as part of the CREATE VIRTUAL TABLE statement used
-**   to create the fts5 table.
+**   was registered with FTS5 (the third argument to xCreateTokenizer()). 
+**   The second and third arguments are an array of nul-terminated strings
+**   containing the tokenizer arguments, if any, specified following the
+**   tokenizer name as part of the CREATE VIRTUAL TABLE statement used
+**   to create the FTS5 table.
 **
 **   The final argument is an output variable. If successful, (*ppOut) 
 **   should be set to point to the new tokenizer handle and SQLITE_OK
@@ -240,12 +241,10 @@ struct Fts5ExtensionApi {
 **   are a pointer to a buffer containing the token text, and the size of
 **   the token in bytes. The 4th and 5th arguments are the byte offsets of
 **   the first byte of and first byte immediately following the text from 
-**   which the token is derived within the input. The final argument is the
-**   token position - the total number of tokens that appear before this one 
-**   in the input buffer.
+**   which the token is derived within the input.
 **
-**   The xToken() callback must be invoked with non-decreasing values of
-**   the iPos parameter.
+**   FTS5 assumes the xToken() callback is invoked for each token in the 
+**   order that they occur within the input text.
 **
 **   If an xToken() callback returns any value other than SQLITE_OK, then
 **   the tokenization should be abandoned and the xTokenize() method should
@@ -256,9 +255,8 @@ struct Fts5ExtensionApi {
 **   SQLITE_OK or SQLITE_DONE.
 **
 */
-typedef struct fts5_tokenizer fts5_tokenizer;
 typedef struct Fts5Tokenizer Fts5Tokenizer;
-
+typedef struct fts5_tokenizer fts5_tokenizer;
 struct fts5_tokenizer {
   int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
   void (*xDelete)(Fts5Tokenizer*);
diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index 0a12adaa4c..3fe34463f1 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -703,7 +703,7 @@ static int fts5ExprNearNextRowidMatch(
 */
 static int fts5ExprNearNextMatch(
   Fts5Expr *pExpr,                /* Expression that pNear is a part of */
-  Fts5ExprNode *pNode,
+  Fts5ExprNode *pNode,            /* The "NEAR" node (FTS5_STRING) */
   int bFromValid,
   i64 iFrom
 ){
@@ -716,6 +716,9 @@ static int fts5ExprNearNextMatch(
     rc = fts5ExprNearNextRowidMatch(pExpr, pNode, bFromValid, iFrom);
     if( pNode->bEof || rc!=SQLITE_OK ) break;
 
+    /* Check that each phrase in the nearset matches the current row.
+    ** Populate the pPhrase->poslist buffers at the same time. If any
+    ** phrase is not a match, break out of the loop early.  */
     for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
       Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
       if( pPhrase->nTerm>1 || pNear->iCol>=0 ){
diff --git a/ext/fts5/test/fts5near.test b/ext/fts5/test/fts5near.test
new file mode 100644
index 0000000000..7425a4f24e
--- /dev/null
+++ b/ext/fts5/test/fts5near.test
@@ -0,0 +1,65 @@
+# 2014 Jan 08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on the NEAR operator.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5near
+
+proc do_near_test {tn doc near res} {
+  uplevel [list do_execsql_test $tn "
+    DELETE FROM t1;
+    INSERT INTO t1 VALUES('$doc');
+    SELECT count(*) FROM t1 WHERE t1 MATCH '$near';
+  " $res]
+}
+
+execsql { 
+  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize = 'simple tokenchars .') 
+}
+
+do_near_test 1.1 ". . a . . . b . ." { NEAR(a b, 5) } 1
+do_near_test 1.2 ". . a . . . b . ." { NEAR(a b, 4) } 1
+do_near_test 1.3 ". . a . . . b . ." { NEAR(a b, 3) } 1
+do_near_test 1.4 ". . a . . . b . ." { NEAR(a b, 2) } 0
+
+do_near_test 1.5 ". . a . . . b . ." { NEAR(b a, 5) } 1
+do_near_test 1.6 ". . a . . . b . ." { NEAR(b a, 4) } 1
+do_near_test 1.7 ". . a . . . b . ." { NEAR(b a, 3) } 1
+do_near_test 1.8 ". . a . . . b . ." { NEAR(b a, 2) } 0
+
+do_near_test 1.9  ". a b . . . c . ." { NEAR("a b" c, 3) } 1
+do_near_test 1.10 ". a b . . . c . ." { NEAR("a b" c, 2) } 0
+do_near_test 1.11 ". a b . . . c . ." { NEAR(c "a b", 3) } 1
+do_near_test 1.12 ". a b . . . c . ." { NEAR(c "a b", 2) } 0
+
+do_near_test 1.13 ". a b . . . c d ." { NEAR(a+b c+d, 3) } 1
+do_near_test 1.14 ". a b . . . c d ." { NEAR(a+b c+d, 2) } 0
+do_near_test 1.15 ". a b . . . c d ." { NEAR(c+d a+b, 3) } 1
+do_near_test 1.16 ". a b . . . c d ." { NEAR(c+d a+b, 2) } 0
+
+do_near_test 1.17 ". a b . . . c d ." { NEAR(a b c d, 5) } 1
+do_near_test 1.18 ". a b . . . c d ." { NEAR(a b c d, 4) } 0
+do_near_test 1.19 ". a b . . . c d ." { NEAR(a+b c d, 4) } 1
+
+do_near_test 1.20 "a b c d e f g h i" { NEAR(b+c a+b+c+d i, 5) } 1
+do_near_test 1.21 "a b c d e f g h i" { NEAR(b+c a+b+c+d i, 4) } 0
+
+do_near_test 1.22 "a b c d e f g h i" { NEAR(a+b+c+d i b+c, 5) } 1
+do_near_test 1.23 "a b c d e f g h i" { NEAR(a+b+c+d i b+c, 4) } 0
+
+do_near_test 1.24 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 5) } 1
+do_near_test 1.25 "a b c d e f g h i" { NEAR(i a+b+c+d b+c, 4) } 0
+
+
+finish_test
+
diff --git a/manifest b/manifest
index adf58144e8..d065fd8d57 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sfts5\s'optimize'\scommand.
-D 2015-01-07T19:33:11.551
+C Fix\ssome\sdocumentation\sissues\sin\sfts5.
+D 2015-01-10T20:34:27.199
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -103,14 +103,14 @@ F ext/fts3/tool/fts3view.c 3986531f2fc0ceca0c89c31ec7d0589b6adb19d6
 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl 4199cb887040ee3c3cd59a5171ddb0566904586e
-F ext/fts5/extract_api_docs.tcl 6320db4a1d0722a4e2069e661381ad75e9889786
+F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
 F ext/fts5/fts5.c c90004f4a91ce4f4dfad2fc980ade0d9314ebb10
-F ext/fts5/fts5.h 0f8563e21ffa69cb87be4c2e24652fc41b441850
+F ext/fts5/fts5.h f931954065693898d26c51f23f1d27200184a69a
 F ext/fts5/fts5Int.h 0142ba4c3c70e1976578604c0e738670f7689726
 F ext/fts5/fts5_aux.c 549aef152b0fd46020f5595d861b1fd60b3f9b4f
 F ext/fts5/fts5_buffer.c 32dd3c950392346ca69a0f1803501766c5c954f9
 F ext/fts5/fts5_config.c 33534ca25198cc62c54ff7d285d455c57ad19399
-F ext/fts5/fts5_expr.c 0320ae948e82cf7dca800463de7f5b6a808ba7c3
+F ext/fts5/fts5_expr.c 6ba7a2e34a80989cca509bd295de1bc9f8e739a3
 F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279
 F ext/fts5/fts5_index.c ea36c1e42aaf8038b6139be95575eb7fe01f34e4
 F ext/fts5/fts5_storage.c 8bc9e5b6654e1545e9513def277ef3f025921664
@@ -135,6 +135,7 @@ F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
 F ext/fts5/test/fts5content.test 4234e0b11e003fe1e80472aa637f70464396fdd0
 F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c
 F ext/fts5/test/fts5fault1.test f3f4c6ed15cc7a4dc8d517c0d1969d8e5a35a65c
+F ext/fts5/test/fts5near.test 70a568a1211a5b6d5a17282790d5f8cbbe086ce0
 F ext/fts5/test/fts5optimize.test 0028c90a7817d3e576d1148fc8dff17d89054e54
 F ext/fts5/test/fts5porter.test 50322599823cb8080a99f0ec0c39f7d0c12bcb5e
 F ext/fts5/test/fts5rebuild.test 2a5e98205393487b4a732c8290999af7c0b907b4
@@ -1273,7 +1274,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 0cb2fed525778d96237b5b0943047665e1f636d1
-R b413984e0011c860316df7bca0fa936a
+P e749be563d8e738af113bd301770e2f22763ab77
+R 5c59d3558d2a230e6048c600760933d7
 U dan
-Z ad35ce36f519fcc615b0ece9f543df9d
+Z 6c17e3ae4cf92b8841424ff4d00c314d
diff --git a/manifest.uuid b/manifest.uuid
index 0a08358e38..de97eaee58 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e749be563d8e738af113bd301770e2f22763ab77
\ No newline at end of file
+512e1bdb4093b59d1494dfc63391476eadd52aea
\ No newline at end of file