From: dan Date: Fri, 12 Apr 2019 20:33:17 +0000 (+0000) Subject: Add the socketvfs test extension. X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f67fbdd1afbbfe0d048fda8c23d36b1df33d82f5;p=thirdparty%2Fsqlite.git Add the socketvfs test extension. FossilOrigin-Name: f5b3ce9404ffc8afdab6d8e57f806afba0b3ee75520d5e03335335d0f93d4a4b --- diff --git a/ext/misc/socketvfs.c b/ext/misc/socketvfs.c new file mode 100644 index 0000000000..95db078fe8 --- /dev/null +++ b/ext/misc/socketvfs.c @@ -0,0 +1,504 @@ +/* +** 2019 April 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** A read-only VFS that reads data from a server instead of a file +** using a custom protocol over a tcp/ip socket. The VFS is named +** "socket". The filename passed to sqlite3_open() is of the +** form "host:portnumber". For example, to connect to the server +** on port 23456 on the localhost: +** +** sqlite3_open_v2("localhost:23456", &db, SQLITE_OPEN_READONLY, "socket"); +** +** Or, if using URIs: +** +** sqlite3_open("file:localhost:23456?vfs=socket", &db); +** +** The protocol is: +** +** * Client connects to tcp/ip server. Server immediately sends the +** database file-size in bytes as a 64-bit big-endian integer. +** +** * To read from the file, client sends the byte offset and amount +** of data required in bytes, both as 64-bit big-endian integers +** (i.e. a 16-byte message). Server sends back the requested data. +** +** As well as the usual SQLite loadable extension entry point, this file +** exports one more function: +** +** sqlite3_vfs *sqlite3_socketvfs(void); +** +** To install the "socket" VFS without loading the extension, link this file +** into the application and invoke: +** +** int bDefault = 0; // Do not make "socket" the default VFS +** sqlite3_vfs_register(sqlite3_socketvfs(), bDefault); +** +*/ + +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +#include +#include + +#if defined(_WIN32) +# if defined(_WIN32_WINNT) +# undef _WIN32_WINNT +# endif +# define _WIN32_WINNT 0x501 +#endif +#ifndef __EXTENSIONS__ +# define __EXTENSIONS__ 1 /* IPv6 won't compile on Solaris without this */ +#endif +#if defined(_WIN32) +# include +# include +# include +# include +#else +# include +# include +# include +# include +# include +#endif +#include +#include +#include + +#if !defined(_WIN32) +# include +#endif + +/* +** When using this VFS, the sqlite3_file* handles that SQLite uses are +** actually pointers to instances of type SocketFile. +*/ +typedef struct SocketFile SocketFile; +struct SocketFile { + sqlite3_file base; /* Base class. Must be first. */ + int iSocket; /* Socket used to talk to server. */ + sqlite3_int64 szFile; /* Size of file in bytes */ +}; + +static sqlite3_uint64 socketGetU64(const unsigned char *a){ + return (((sqlite3_uint64)(a[0])) << 56) + + (((sqlite3_uint64)(a[1])) << 48) + + (((sqlite3_uint64)(a[2])) << 40) + + (((sqlite3_uint64)(a[3])) << 32) + + (((sqlite3_uint64)(a[4])) << 24) + + (((sqlite3_uint64)(a[5])) << 16) + + (((sqlite3_uint64)(a[6])) << 8) + + (((sqlite3_uint64)(a[7])) << 0); +} + +static void socketPutU64(unsigned char *a, sqlite3_int64 i){ + a[0] = ((i >> 56) & 0xFF); + a[1] = ((i >> 48) & 0xFF); + a[2] = ((i >> 40) & 0xFF); + a[3] = ((i >> 32) & 0xFF); + a[4] = ((i >> 24) & 0xFF); + a[5] = ((i >> 16) & 0xFF); + a[6] = ((i >> 8) & 0xFF); + a[7] = ((i >> 0) & 0xFF); +} + +static void socket_close(int iSocket){ + if( iSocket>=0 ){ +#if defined(_WIN32) + if( shutdown(iSocket,1)==0 ) shutdown(iSocket,0); + closesocket(iSocket); +#else + close(iSocket); +#endif + } +} + +/* +** Write nData bytes of data from buffer aData to socket iSocket. If +** successful, return SQLITE_OK. Otherwise, SQLITE_IOERR_WRITE. +*/ +static int socket_send(int iSocket, const unsigned char *aData, int nData){ + int nWrite = 0; + do{ + int res = send(iSocket, (const char*)&aData[nWrite], nData-nWrite, 0); + if( res<=0 ) return SQLITE_IOERR_WRITE; + nWrite += res; + }while( nWriteiSocket); + pSock->iSocket = -1; + return SQLITE_OK; +} + +/* +** Read data from a SocketFile file. +*/ +static int socketRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite3_int64 iOfst +){ + SocketFile *pSock = (SocketFile*)pFile; + unsigned char aRequest[16]; + int rc = SQLITE_OK; + int nRead = iAmt; + + if( iOfst+nRead>pSock->szFile ){ + nRead = (int)(pSock->szFile - iOfst); + memset(zBuf, 0, iAmt); + rc = SQLITE_IOERR_SHORT_READ; + } + + if( nRead>0 ){ + socketPutU64(&aRequest[0], (sqlite3_uint64)iOfst); + socketPutU64(&aRequest[8], (sqlite3_uint64)nRead); + rc = socket_send(pSock->iSocket, aRequest, sizeof(aRequest)); + if( rc==SQLITE_OK ){ + rc = socket_recv(pSock->iSocket, zBuf, nRead); + } + } + + return rc; +} + +/* +** Write to a file. This is a no-op, as this VFS is always opens files +** read-only. +*/ +static int socketWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite3_int64 iOfst +){ + return SQLITE_IOERR_WRITE; +} + +/* +** Truncate a file. This is a no-op, as this VFS is always opens files +** read-only. +*/ +static int socketTruncate(sqlite3_file *pFile, sqlite3_int64 size){ + return SQLITE_IOERR_TRUNCATE; +} + +/* +** Synk a file. This is a no-op, as this VFS is always opens files +** read-only. +*/ +static int socketSync(sqlite3_file *pFile, int flags){ + return SQLITE_IOERR_FSYNC; +} + +/* +** Write the size of the file in bytes to *pSize. +*/ +static int socketFileSize(sqlite3_file *pFile, sqlite3_int64 *pSize){ + SocketFile *pSock = (SocketFile*)pFile; + *pSize = pSock->szFile; + return SQLITE_OK; +} + +/* +** Locking functions. All no-ops. +*/ +static int socketLock(sqlite3_file *pFile, int eLock){ + return SQLITE_OK; +} +static int socketUnlock(sqlite3_file *pFile, int eLock){ + return SQLITE_OK; +} +static int socketCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + *pResOut = 0; + return SQLITE_OK; +} + +/* +** No xFileControl() verbs are implemented by this VFS. +*/ +static int socketFileControl(sqlite3_file *pFile, int op, void *pArg){ + return SQLITE_OK; +} + +/* +** The xSectorSize() and xDeviceCharacteristics() methods. These two +** may return special values allowing SQLite to optimize file-system +** access to some extent. But it is also safe to simply return 0. +*/ +static int socketSectorSize(sqlite3_file *pFile){ + return 0; +} +static int socketDeviceCharacteristics(sqlite3_file *pFile){ + return 0; +} + +/* +** Open a SocketFile file. +*/ +static int socketOpen( + sqlite3_vfs *pVfs, /* VFS */ + const char *zName, /* File to open, or 0 for a temp file */ + sqlite3_file *pFile, /* Pointer to SocketFile struct to populate */ + int flags, /* Input SQLITE_OPEN_XXX flags */ + int *pOutFlags /* Output SQLITE_OPEN_XXX flags (or NULL) */ +){ + static const sqlite3_io_methods socketio = { + 1, /* iVersion */ + socketClose, /* xClose */ + socketRead, /* xRead */ + socketWrite, /* xWrite */ + socketTruncate, /* xTruncate */ + socketSync, /* xSync */ + socketFileSize, /* xFileSize */ + socketLock, /* xLock */ + socketUnlock, /* xUnlock */ + socketCheckReservedLock, /* xCheckReservedLock */ + socketFileControl, /* xFileControl */ + socketSectorSize, /* xSectorSize */ + socketDeviceCharacteristics /* xDeviceCharacteristics */ + }; + + SocketFile *pSock = (SocketFile*)pFile; + + char zHost[1024]; + const char *zPort; + int i; + + struct addrinfo hints; + struct addrinfo *ai = 0; + struct addrinfo *pInfo; + unsigned char aFileSize[8]; + + pSock->iSocket = -1; + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN; + + /* Parse the argument and copy the results to zHost and zPort. It should be + ** "hostname:port". Anything else is an error. */ + assert( sizeof(zHost)>=pVfs->mxPathname ); + if( zName==0 ) return SQLITE_CANTOPEN; + for(i=0; zName[i] && zName[i]!=':'; i++); + if( zName[i]==0 ) return SQLITE_CANTOPEN; + memcpy(zHost, zName, i); + zHost[i] = '\0'; + zPort = &zName[i+1]; + + /* Resolve the address */ + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = IPPROTO_TCP; + if( getaddrinfo(zHost, zPort, &hints, &ai) ){ + return SQLITE_CANTOPEN; + } + + /* Connect to the resolved address. Set SocketFile.iSocket to the tcp/ip + ** socket and return SQLITE_OK. */ + for(pInfo=ai; pInfo; pInfo=pInfo->ai_next){ + int sd = socket(pInfo->ai_family, pInfo->ai_socktype, pInfo->ai_protocol); + if( sd<0 ) continue; + if( connect(sd, pInfo->ai_addr, pInfo->ai_addrlen)<0 ){ + socket_close(sd); + continue; + } + pSock->iSocket = sd; + break; + } + + if( ai ) freeaddrinfo(ai); + if( pSock->iSocket<0 ) return SQLITE_CANTOPEN; + + /* The server sends back the file size as a 64-bit big-endian */ + if( socket_recv(pSock->iSocket, aFileSize, 8) ){ + socket_close(pSock->iSocket); + return SQLITE_CANTOPEN; + } + pSock->szFile = (sqlite3_int64)socketGetU64(aFileSize); + + *pOutFlags = flags & ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); + *pOutFlags |= SQLITE_OPEN_READONLY; + pSock->base.pMethods = &socketio; + return SQLITE_OK; +} + +/* +** Another no-op. This is a read-only VFS. +*/ +static int socketDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return SQLITE_IOERR_DELETE; +} + +/* +** This is used by SQLite to detect journal and wal files. Which cannot +** exist for this VFS. So always set the output to false and return +** SQLITE_OK. +*/ +static int socketAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + *pResOut = 0; + return SQLITE_OK; +} + +/* +** A no-op. Copy the input to the output. +*/ +static int socketFullPathname( + sqlite3_vfs *pVfs, /* VFS */ + const char *zPath, /* Input path (possibly a relative path) */ + int nPathOut, /* Size of output buffer in bytes */ + char *zPathOut /* Pointer to output buffer */ +){ + int nByte = strlen(zPath); + if( nByte>=pVfs->mxPathname ) return SQLITE_IOERR; + memcpy(zPathOut, zPath, nByte+1); + return SQLITE_OK; +} + +/* +** The following four VFS methods: +** +** xDlOpen +** xDlError +** xDlSym +** xDlClose +** +** are supposed to implement the functionality needed by SQLite to load +** extensions compiled as shared objects. This simple VFS does not support +** this functionality, so the following functions are no-ops. +*/ +static void *socketDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return 0; +} +static void socketDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + sqlite3_snprintf(nByte, zErrMsg, "Loadable extensions are not supported"); + zErrMsg[nByte-1] = '\0'; +} +static void (*socketDlSym(sqlite3_vfs *pVfs, void *pH, const char *z))(void){ + return 0; +} +static void socketDlClose(sqlite3_vfs *pVfs, void *pHandle){ + return; +} + +/* +** Parameter zByte points to a buffer nByte bytes in size. Populate this +** buffer with pseudo-random data. +*/ +static int socketRandomness(sqlite3_vfs *pVfs, int nByte, char *zByte){ + memset(zByte, 0, nByte); + return SQLITE_OK; +} + +/* +** Sleep for at least nMicro microseconds. Return the (approximate) number +** of microseconds slept for. +*/ +static int socketSleep(sqlite3_vfs *pVfs, int nMicro){ +#ifdef _WIN32 + Sleep(nMicro/1000); +#else + sleep(nMicro / 1000000); + usleep(nMicro % 1000000); +#endif + return nMicro; +} + +/* +** Set *pTime to the current UTC time expressed as a Julian day. Return +** SQLITE_OK if successful, or an error code otherwise. +** +** http://en.wikipedia.org/wiki/Julian_day +** +** This implementation is not very good. The current time is rounded to +** an integer number of seconds. Also, assuming time_t is a signed 32-bit +** value, it will stop working some time in the year 2038 AD (the so-called +** "year 2038" problem that afflicts systems that store time this way). +*/ +static int socketCurrentTime(sqlite3_vfs *pVfs, double *pTime){ + time_t t = time(0); + *pTime = t/86400.0 + 2440587.5; + return SQLITE_OK; +} + +/* +** This function returns a pointer to the VFS implemented in this file. +** To make the VFS available to SQLite: +** +** sqlite3_vfs_register(sqlite3_socketvfs(), 0); +*/ +sqlite3_vfs *sqlite3_socketvfs(void){ + static sqlite3_vfs socketvfs = { + 1, /* iVersion */ + sizeof(SocketFile), /* szOsFile */ + 512, /* mxPathname */ + 0, /* pNext */ + "socket", /* zName */ + 0, /* pAppData */ + socketOpen, /* xOpen */ + socketDelete, /* xDelete */ + socketAccess, /* xAccess */ + socketFullPathname, /* xFullPathname */ + socketDlOpen, /* xDlOpen */ + socketDlError, /* xDlError */ + socketDlSym, /* xDlSym */ + socketDlClose, /* xDlClose */ + socketRandomness, /* xRandomness */ + socketSleep, /* xSleep */ + socketCurrentTime, /* xCurrentTime */ + }; + return &socketvfs; +} + +/* +** Register the amatch virtual table +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_socketvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Not used */ + sqlite3_vfs_register(sqlite3_socketvfs(), 0); + return SQLITE_OK_LOAD_PERMANENTLY; +} + diff --git a/ext/misc/socketvfs_server.tcl b/ext/misc/socketvfs_server.tcl new file mode 100644 index 0000000000..ebb4ddfeca --- /dev/null +++ b/ext/misc/socketvfs_server.tcl @@ -0,0 +1,52 @@ +#!/bin/sh +# \ +exec tclsh "$0" ${1+"$@"} + +if {[llength $argv]!=2} { + puts stderr "Usage: $argv0 " + exit -1 +} +set G(filename) [lindex $argv 0] +set G(port) [lindex $argv 1] + +proc new_message {chan} { + global G + if {[eof $chan]} { + close $chan + puts "Close channel $chan" + } else { + set msg [read $chan 16] + if {[string length $msg]>0} { + binary scan $msg WW offset amt + # puts "Request from $chan for $amt bytes at offset $offset" + seek $G(fd) $offset + set data [read $G(fd) $amt] + puts -nonewline $chan $data + flush $chan + } + } +} + +proc new_connection {chan addr port} { + global G + + set sz [file size $G(filename)] + + puts -nonewline "$addr:$port connects! " + puts "Sending file size ($sz) as a 64-bit big-endian integer." + set bin [binary format W $sz] + puts -nonewline $chan $bin + flush $chan + + fconfigure $chan -encoding binary + fconfigure $chan -translation binary + fileevent $chan readable [list new_message $chan] +} + +set G(fd) [open $G(filename) r] +fconfigure $G(fd) -encoding binary +fconfigure $G(fd) -translation binary + +socket -server new_connection $G(port) +vwait forever + diff --git a/manifest b/manifest index 9c55f2f0de..00fac53f99 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scase\schanges\sso\sthat\sthey\swork\swith\sboth\sTcl8.6\sand\sTcl8.7. -D 2019-04-12T16:25:42.811 +C Add\sthe\ssocketvfs\stest\sextension. +D 2019-04-12T20:33:17.983 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -308,6 +308,8 @@ F ext/misc/series.c 0c97f63378fddc9f425e82ba139b9aaf902211f24ced115c2b6ae12b425f F ext/misc/sha1.c df0a667211baa2c0612d8486acbf6331b9f8633fd4d605c17c7cccd26d59c6bd F ext/misc/shathree.c 22ba7ca84a433d6466a7d05dcc876910b435a715da8cc462517db9351412b8c8 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 +F ext/misc/socketvfs.c 07f4e382d2c2c175177b55189767d2a6a0eba14565a12f4fdb02e9b486d96259 +F ext/misc/socketvfs_server.tcl 370a38231f255439a219ff68c8cfdfa03c57d39031b4aa8a7b87906e9d3681e0 F ext/misc/spellfix.c f88ecb2c0294453ce8b7704b211f5350c41b085b38c8e056852e3a08b0f5e484 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74 F ext/misc/stmt.c 8a8dc4675042e4551e4afe99b8d0cc7a4a2fc1a8dacc0a9ce1b1bbff145da93d @@ -1818,7 +1820,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dd248c186a157b5ed60bbdae669b31237b8354c1a7e6ed64d377904043bd57ee -R b3f8e9d256ef93f75dfd1b764c21d6e8 -U drh -Z 29d22b4357a8e51c272b10c19deb4f0a +P 7b771405a9adc3ec191156be4ebe7122f4c698d88d69ae2134c75acb8d8feebb +R b39fe5513a86675b826ef91bc6fba12d +T *branch * socketvfs +T *sym-socketvfs * +T -sym-trunk * +U dan +Z 3ab2817f46fa2cfe678b228c9c20911b diff --git a/manifest.uuid b/manifest.uuid index 38493de4a7..b2e695ba0a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7b771405a9adc3ec191156be4ebe7122f4c698d88d69ae2134c75acb8d8feebb \ No newline at end of file +f5b3ce9404ffc8afdab6d8e57f806afba0b3ee75520d5e03335335d0f93d4a4b \ No newline at end of file