From: larrybr Date: Mon, 21 Nov 2022 00:11:09 +0000 (+0000) Subject: Fix a base64 decode bug. Provide for convenient inclusion of extension(s) built into... X-Git-Tag: version-3.41.0~363^2~7 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=423003dca91a84409950bfd4e99ee68dc4f831ee;p=thirdparty%2Fsqlite.git Fix a base64 decode bug. Provide for convenient inclusion of extension(s) built into the CLI, to simplify testing and for its own sake. Improve comments. Cure collision between base64.c and base85.c when both are in the same translation unit. FossilOrigin-Name: 07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f --- diff --git a/ext/misc/base64.c b/ext/misc/base64.c index 1ed1a4f174..d6f7834a85 100644 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -12,7 +12,7 @@ ** ** This is a SQLite extension for converting in either direction ** between a (binary) blob and base64 text. Base64 can transit a -** sane ASCII channel unmolested. It also plays nicely in CSV or +** sane USASCII channel unmolested. It also plays nicely in CSV or ** written as TCL brace-enclosed literals or SQL string literals, ** and can be used unmodified in XML-like documents. ** @@ -37,10 +37,10 @@ ** error will be thrown for other input argument types. ** ** This code relies on UTF-8 encoding only with respect to the -** meaning of the first 128 (7-bit) codes being the same as ASCII. +** meaning of the first 128 (7-bit) codes matching that of USASCII. ** It will fail miserably if somehow made to try to convert EBCDIC. -** Because it is table-driven, it could be enhanced to handle that. -** But the world and SQLite have moved on from that anachronism. +** Because it is table-driven, it could be enhanced to handle that, +** but the world and SQLite have moved on from that anachronism. ** ** To build the extension: ** Set shell variable SQDIR= @@ -52,12 +52,16 @@ #include +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ #include "sqlite3ext.h" +#endif + SQLITE_EXTENSION_INIT1; #define PC 0x80 /* pad character */ #define WS 0x81 /* whitespace */ #define ND 0x82 /* Not above or digit-value */ +#define PAD_CHAR '=' typedef unsigned char ubyte; @@ -89,10 +93,10 @@ static const char b64Numerals[64] #define IS_BX_PAD(bdp) ((bdp)==PC) #define BX_NUMERAL(dv) (b64Numerals[dv]) /* Width of base64 lines. Should be an integer multiple of 4. */ -#define DARK_MAX 72 +#define B64_DARK_MAX 72 /* Encode a byte buffer into base64 text. If pSep!=0, it's a C string -** to be appended to encoded groups to limit their length to DARK_MAX +** to be appended to encoded groups to limit their length to B64_DARK_MAX ** or to terminate the last group (to aid concatenation.) */ static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ @@ -111,12 +115,12 @@ static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ nc = ncio[nbi]; nbIn -= nbi; for( nbe=3; nbe>=0; --nbe ){ - char ce = (nbe0 && *pIn!='=' ){ + while( ncIn>0 && *pIn!=PAD_CHAR ){ static signed char nboi[] = { 0, 0, 1, 2, 3 }; char *pUse = skipNonB64(pIn); unsigned long qv = 0L; - int nti, nbo; + int nti, nbo, nac; ncIn -= (pUse - pIn); - if( ncIn<=0 ) break; pIn = pUse; nti = (ncIn>4)? 4 : ncIn; nbo = nboi[nti]; - while( nti>0 ){ - char c = *pIn++; + if( nbo==0 ) break; + for( nac=0; nac<4; ++nac ){ + char c = (nac<=nti)? *pIn++ : PAD_CHAR; ubyte bdp = BX_DV_PROTO(c); --ncIn; switch( bdp ){ @@ -157,15 +161,16 @@ static ubyte* fromBase64( char *pIn, int ncIn, ubyte *pOut ){ break; case PC: bdp = 0; + --nbo; /* fall thru */ - default: /* It's the digit value. */ + default: /* bdp is the digit value. */ qv = qv<<6 | bdp; - --nti; break; } } + nti = 2; while( nbo-- > 0 ){ - *pOut++ = (qv >> (8*nbo))&0xff; + *pOut++ = (qv >> (8*nti--))&0xff; } } return pOut; @@ -183,7 +188,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ case SQLITE_BLOB: nb = nv; nc = 4*(nv+2/3); /* quads needed */ - nc += (nc+(DARK_MAX-1))/DARK_MAX + 1; /* LFs and a 0-terminator */ + nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ if( nvMax < nc ){ sqlite3_result_error(context, "blob expanded to base64 too big.", -1); } @@ -219,11 +224,15 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ /* ** Establish linkage to running SQLite library. */ +#ifndef SQLITE_SHELL_EXTFUNCS #ifdef _WIN32 __declspec(dllexport) #endif -int sqlite3_base_init(sqlite3 *db, char **pzErr, - const sqlite3_api_routines *pApi){ +int sqlite3_base_init +#else +static int sqlite3_base64_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErr; return sqlite3_create_function @@ -231,3 +240,11 @@ int sqlite3_base_init(sqlite3 *db, char **pzErr, SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, 0, base64, 0, 0); } + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +#define BASE64_INIT(db) sqlite3_base64_init(db, 0, 0) +#define BASE64_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ diff --git a/ext/misc/base85.c b/ext/misc/base85.c index 365db80f2b..dfa9bd0e31 100644 --- a/ext/misc/base85.c +++ b/ext/misc/base85.c @@ -14,7 +14,7 @@ ** It can be built as a standalone program or an SQLite3 extension. ** ** Much like base64 representations, base85 can be sent through a -** sane ASCII channel unmolested. It also plays nicely in CSV or +** sane USASCII channel unmolested. It also plays nicely in CSV or ** written as TCL brace-enclosed literals or SQL string literals. ** It is not suited for unmodified use in XML-like documents. ** @@ -24,7 +24,7 @@ ** Further, this is an independent implementation of a base85 system. ** Hence, the author has rightfully put this into the public domain. ** -** Base85 numerals are taken from the set of 7-bit ASCII codes, +** Base85 numerals are taken from the set of 7-bit USASCII codes, ** excluding control characters and Space ! " ' ( ) { | } ~ Del ** in code order representing digit values 0 to 84 (base 10.) ** @@ -32,8 +32,11 @@ ** are represented as 5-digit base85 numbers with MS to LS digit ** order. Groups of 1-3 bytes are represented with 2-4 digits, ** still big-endian but 8-24 bit values. (Using big-endian yields -** the simplest transition to byte groups smaller than 4 bytes.) +** the simplest transition to byte groups smaller than 4 bytes. +** These byte groups can also be considered base-256 numbers.) ** Groups of 0 bytes are represented with 0 digits and vice-versa. +** No pad characters are used; Encoded base85 numeral sequence +** (aka "group") length maps 1-to-1 to the decoded binary length. ** ** Any character not in the base85 numeral set delimits groups. ** When base85 is streamed or stored in containers of indefinite @@ -82,7 +85,10 @@ #ifndef BASE85_STANDALONE +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ # include "sqlite3ext.h" +#endif + SQLITE_EXTENSION_INIT1; #else @@ -107,7 +113,7 @@ static void sayHelp(){ } #endif -/* Classify c according to interval within ASCII set w.r.t. base85 +/* Classify c according to interval within USASCII set w.r.t. base85 * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. */ #define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z')) @@ -127,16 +133,25 @@ static unsigned char base85DigitValue( char c ){ } #endif +/* Width of base64 lines. Should be an integer multiple of 5. */ +#define B85_DARK_MAX 80 + + static char * skipNonB85( char *s ){ char c; while( (c = *s) && !IS_B85(c) ) ++s; return s; } +/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral.*/ static char base85Numeral( unsigned char b ){ return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); } +/* Encode a byte buffer into base85 text. If pSep!=0, it's a C string +** to be appended to encoded groups to limit their length to B85_DARK_MAX +** or to terminate the last group (to aid concatenation.) +*/ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ int nCol = 0; *pOut = 0; @@ -157,7 +172,7 @@ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ pOut[--nco] = base85Numeral(dv); } pOut += ncio[nbi]; - if( pSep && ((nCol += ncio[nbi])>=80 || nbIn<=0) ){ + if( pSep && ((nCol += ncio[nbi])>=B85_DARK_MAX || nbIn<=0) ){ char *p = pSep; while( *p ) *pOut++ = *p++; nCol = 0; @@ -167,6 +182,7 @@ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ return pOut; } +/* Decode base85 text into a byte buffer. */ static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){ if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; while( ncIn>0 ){ @@ -196,6 +212,7 @@ static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){ } #ifndef OMIT_BASE85_CHECKER +/* Say whether input char sequence is all (base85 and/or whitespace).*/ static int allBase85( char *p, int len ){ char c; while( len-- > 0 && (c = *p++) != 0 ){ @@ -208,6 +225,7 @@ static int allBase85( char *p, int len ){ #ifndef BASE85_STANDALONE # ifndef OMIT_BASE85_CHECKER +/* This function does the work for the SQLite is_base85(t) UDF. */ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ assert(na==1); switch( sqlite3_value_type(av[0]) ){ @@ -228,6 +246,7 @@ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ } # endif +/* This function does the work for the SQLite base85(x) UDF. */ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ int nb, nc, nv = sqlite3_value_bytes(av[0]); int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), @@ -272,11 +291,18 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_error(context, "base85 OOM", -1); } +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS #ifdef _WIN32 __declspec(dllexport) #endif -int sqlite3_base_init(sqlite3 *db, char **pzErr, - const sqlite3_api_routines *pApi){ +int sqlite3_base_init +#else +static int sqlite3_base85_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErr; # ifndef OMIT_BASE85_CHECKER @@ -294,12 +320,20 @@ int sqlite3_base_init(sqlite3 *db, char **pzErr, 0, base85, 0, 0); } +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +# define BASE85_INIT(db) sqlite3_base85_init(db, 0, 0) +# define BASE85_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ + #else /* standalone program */ int main(int na, char *av[]){ int cin; int rc = 0; - unsigned char bBuf[64]; + unsigned char bBuf[4*(B85_DARK_MAX/5)]; char cBuf[5*(sizeof(bBuf)/4)+2]; size_t nio; # ifndef OMIT_BASE85_CHECKER diff --git a/ext/misc/basexx.c b/ext/misc/basexx.c new file mode 100644 index 0000000000..700a99409c --- /dev/null +++ b/ext/misc/basexx.c @@ -0,0 +1,64 @@ +/* +** 2022-11-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This source allows multiple SQLite extensions to be either: combined +** into a single runtime-loadable library; or built into the SQLite shell +** using a preprocessing convention set by src/shell.c.in (and shell.c). +** +** Presently, it combines the base64.c and base85.c extensions. However, +** it can be used as a template for other combinations. +*/ + +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ +# include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1; +#endif + +static void init_api_ptr(const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); +} + +#undef SQLITE_EXTENSION_INIT1 +#define SQLITE_EXTENSION_INIT1 /* */ +#undef SQLITE_EXTENSION_INIT2 +#define SQLITE_EXTENSION_INIT2(v) (void)v + +/* These next 2 undef's are only needed because the entry point names + * collide when formulated per the rules stated for loadable extension + * entry point names that will be deduced from the file basenames. + */ +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base64_init +#include "base64.c" + +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base85_init +#include "base85.c" + +static int sqlite3_basexx_init(sqlite3 *db, char **pzErr, + const sqlite3_api_routines *pApi){ + init_api_ptr(pApi); + int rc1 = BASE64_INIT(db); + int rc2 = BASE85_INIT(db); + int rc = SQLITE_OK; + + if( rc1==SQLITE_OK && rc2==SQLITE_OK ){ + BASE64_EXPOSE(db, pzErr); + BASE64_EXPOSE(db, pzErr); + return SQLITE_OK; + }else{ + return SQLITE_ERROR; + } +} + +# define BASEXX_INIT(db) sqlite3_basexx_init(db, 0, 0) +# define BASEXX_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ diff --git a/manifest b/manifest index faf2f3f49e..f02b4834b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\sextensions\sfor\sbase85\sand\sbase64\sconversion\sUDFs -D 2022-11-19T02:39:16.296 +C Fix\sa\sbase64\sdecode\sbug.\sProvide\sfor\sconvenient\sinclusion\sof\sextension(s)\sbuilt\sinto\sthe\sCLI,\sto\ssimplify\stesting\sand\sfor\sits\sown\ssake.\sImprove\scomments.\sCure\scollision\sbetween\sbase64.c\sand\sbase85.c\swhen\sboth\sare\sin\sthe\ssame\stranslation\sunit. +D 2022-11-21T00:11:09.323 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,8 +289,9 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c e4d3f13bb59aa903734b557e1400f3c5961f0a5abc40400e0e2aa3ad362c0fd7 -F ext/misc/base85.c 3e07aea038129ee646b129ac2ed736344ea03859ff447019e67250b80722c528 +F ext/misc/base64.c 05da915d991f24e59515d0566f9a206c338032410d42472a3d9348da8bc166f9 +F ext/misc/base85.c 2c680ca7733f9a86f5d292fec71d10777290e68e7ae59d90597ae75fc44a88b6 +F ext/misc/basexx.c d32037f1414d9da11e0be334395cd4d82438b756933c99fc4075b6346cd11fd2 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69 @@ -640,7 +641,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 9886d6669f5787471aab6ae52af76fad90b53edb1c218fc9ed9d953363bc5184 -F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d +F src/shell.c.in 49b6aeaf950b9b7a74aca7b80e6d004ed32f0a16d2c9471e34587cc4755d5eaa F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2057,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea -R 4ecad003b3a6f60dd6b11d354f8edbe2 +P 5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 +R 0260a27c5340f9b412a0f6e0981a0d6d U larrybr -Z d182470ca73df70df09238ac5a5e4382 +Z 18487c3a1bcdc27a89418b38ad3e3f97 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b47c8a0962..41284b2a7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 \ No newline at end of file +07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index db8d987660..478c055357 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1070,6 +1070,9 @@ INCLUDE ../ext/recover/dbdata.c INCLUDE ../ext/recover/sqlite3recover.h INCLUDE ../ext/recover/sqlite3recover.c #endif +#ifdef SQLITE_SHELL_EXTSRC +# include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) +#endif #if defined(SQLITE_ENABLE_SESSION) /* @@ -5115,6 +5118,7 @@ static void open_db(ShellState *p, int openFlags){ } exit(1); } + #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); #endif @@ -5137,6 +5141,34 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_sqlar_init(p->db, 0, 0); } #endif +#ifdef SQLITE_SHELL_EXTFUNCS + /* Create a preprocessing mechanism for extensions to make + * their own provisions for being built into the shell. + * This is a short-span macro. See further below for usage. + */ +#define SHELL_SUB_MACRO(base, variant) base ## _ ## variant +#define SHELL_SUBMACRO(base, variant) SHELL_SUB_MACRO(base, variant) + /* Let custom-included extensions get their ..._init() called. + * The WHATEVER_INIT( db, pzErrorMsg, pApi ) macro should cause + * the extension's sqlite3_*_init( db, pzErrorMsg, pApi ) + * inititialization routine to be called. + */ + { + int irc = SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, INIT)(p->db); + /* Let custom-included extensions expose their functionality. + * The WHATEVER_EXPOSE( db, pzErrorMsg ) macro should cause + * the SQL functions, virtual tables, collating sequences or + * VFS's implemented by the extension to be registered. + */ + if( irc==SQLITE_OK + || irc==SQLITE_OK_LOAD_PERMANENTLY ){ + SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, EXPOSE)(p->db, 0); + } +#undef SHELL_SUB_MACRO +#undef SHELL_SUBMACRO + } +#endif + sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0, @@ -5157,6 +5189,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0, editFunc, 0, 0); #endif + if( p->openMode==SHELL_OPEN_ZIPFILE ){ char *zSql = sqlite3_mprintf( "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", zDbFilename);