From: stephan Date: Sat, 19 Aug 2023 10:43:05 +0000 (+0000) Subject: Add multi-thread run mode to JNI Tester1. It works but hangs on exit sometimes for... X-Git-Tag: version-3.44.0~305^2~23 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3cf6c0f276cd39e0dbac1497c57e7e26af26b486;p=thirdparty%2Fsqlite.git Add multi-thread run mode to JNI Tester1. It works but hangs on exit sometimes for Java reasons as yet not understood. FossilOrigin-Name: bdbaf7a4534f40e550b646979e67e7b7731566bb5a2631ed376ac85a9bec40a7 --- diff --git a/ext/jni/GNUmakefile b/ext/jni/GNUmakefile index 22301a3245..6227be435b 100644 --- a/ext/jni/GNUmakefile +++ b/ext/jni/GNUmakefile @@ -163,14 +163,11 @@ SQLITE_OPT = \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_OMIT_DEPRECATED \ -DSQLITE_OMIT_SHARED_CACHE \ - -DSQLITE_THREADSAFE=0 \ + -DSQLITE_THREADSAFE=1 \ -DSQLITE_TEMP_STORE=2 \ -DSQLITE_USE_URI=1 \ -DSQLITE_C=$(sqlite3.c) \ -DSQLITE_DEBUG -# -DSQLITE_DEBUG is just to work around a -Wall warning -# for a var which gets set in all builds but only read -# via assert(). SQLITE_OPT += -g -DDEBUG -UNDEBUG @@ -223,7 +220,7 @@ $(sqlite3-jni.h): $(sqlite3-jni.h.in) $(MAKEFILE) $(sqlite3-jni.dll): $(sqlite3-jni.h) $(sqlite3.c) $(sqlite3.h) $(sqlite3-jni.dll): $(dir.bld.c) $(sqlite3-jni.c) $(SQLite3Jni.java) $(MAKEFILE) $(CC) $(sqlite3-jni.dll.cflags) $(SQLITE_OPT) \ - $(sqlite3-jni.c) -shared -o $@ + $(sqlite3-jni.c) -shared -o $@ -lpthread all: $(sqlite3-jni.dll) .PHONY: test @@ -231,7 +228,7 @@ test.flags ?= -v test: $(SQLite3Jni.class) $(sqlite3-jni.dll) $(bin.java) -ea -Djava.library.path=$(dir.bld.c) \ $(java.flags) -cp $(classpath) \ - org.sqlite.jni.Tester1 $(if $(test.flags),-- $(test.flags),) + org.sqlite.jni.Tester1 $(test.flags) tester.scripts := $(sort $(wildcard $(dir.src)/tests/*.test)) tester.flags ?= # --verbose diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c index 3595eba0aa..ebd1e7940c 100644 --- a/ext/jni/src/c/sqlite3-jni.c +++ b/ext/jni/src/c/sqlite3-jni.c @@ -559,9 +559,9 @@ static struct { #define MUTEX_ASSERT_NOTLOCKER_ENV \ assert( (env) != S3JniGlobal.envCache.locker && "Misuse of S3JniGlobal.envCache.mutex" ) #define MUTEX_ENV_ENTER \ - /*MARKER(("Entering ENV mutex@%p %s.\n", env, __func__));*/ \ MUTEX_ASSERT_NOTLOCKER_ENV; \ sqlite3_mutex_enter( S3JniGlobal.envCache.mutex ); \ + /*MARKER(("Entered ENV mutex@%p %s.\n", env, __func__));*/ \ ++S3JniGlobal.metrics.nMutexEnv; \ S3JniGlobal.envCache.locker = env #define MUTEX_ENV_LEAVE \ @@ -637,6 +637,8 @@ static S3JniEnv * S3JniGlobal_env_cache(JNIEnv * const env){ S3JniGlobal.envCache.aHead = row; row->env = env; + //MARKER(("Initalizing cache for JNIEnv@%p\n", env)); + /* Grab references to various global classes and objects... */ row->g.cObj = REF_G((*env)->FindClass(env,"java/lang/Object")); EXCEPTION_IS_FATAL("Error getting reference to Object class."); @@ -918,7 +920,6 @@ static void S3JniDb_set_aside(S3JniDb * const s){ if(s){ JNIEnv * const env = s->env; MUTEX_ASSERT_LOCKED_PDB; - assert(s->pDb && "Else this object is already in the free-list."); //MARKER(("state@%p for db@%p setting aside\n", s, s->pDb)); assert(s->pPrev != s); assert(s->pNext != s); @@ -966,7 +967,9 @@ static void S3JniDb_free_for_env(JNIEnv *env){ for( ; ps; ps = pNext ){ pNext = ps->pNext; if(ps->env == env){ +#ifndef NDEBUG S3JniDb * const pPrev = ps->pPrev; +#endif S3JniDb_set_aside(ps); assert( pPrev ? pPrev->pNext==pNext : 1 ); assert( ps == S3JniGlobal.perDb.aFree ); @@ -996,6 +999,7 @@ static int S3JniGlobal_env_uncache(JNIEnv * const env){ if( !row ){ return 0; } + //MARKER(("Uncaching JNIEnv@%p\n", env)); if( row->pNext ) row->pNext->pPrev = row->pPrev; if( row->pPrev ) row->pPrev->pNext = row->pNext; if( S3JniGlobal.envCache.aHead == row ){ @@ -2104,11 +2108,13 @@ static jint s3jni_close_db(JNIEnv * const env, jobject jDb, int version){ ps = S3JniDb_for_db(env, jDb, 0); if(ps){ rc = 1==version ? (jint)sqlite3_close(ps->pDb) : (jint)sqlite3_close_v2(ps->pDb); - MUTEX_PDB_ENTER; - S3JniDb_set_aside(ps) - /* MUST come after close() because of ps->trace. */; - MUTEX_PDB_LEAVE; - NativePointerHolder_set(env, jDb, 0, &S3NphRefs.sqlite3); + if( 0==rc ){ + MUTEX_PDB_ENTER; + S3JniDb_set_aside(ps) + /* MUST come after close() because of ps->trace. */; + MUTEX_PDB_LEAVE; + NativePointerHolder_set(env, jDb, 0, &S3NphRefs.sqlite3); + } } return (jint)rc; } @@ -4391,6 +4397,10 @@ Java_org_sqlite_jni_SQLite3Jni_init(JENV_CSELF){ jfieldID fieldId; const ConfigFlagEntry * pConfFlag; + if( 0==sqlite3_threadsafe() ){ + (*env)->FatalError(env, "sqlite3 was not built with SQLITE_THREADSAFE."); + return; + } memset(&S3JniGlobal, 0, sizeof(S3JniGlobal)); if( (*env)->GetJavaVM(env, &S3JniGlobal.jvm) ){ (*env)->FatalError(env, "GetJavaVM() failure shouldn't be possible."); diff --git a/ext/jni/src/org/sqlite/jni/OutputPointer.java b/ext/jni/src/org/sqlite/jni/OutputPointer.java index 82a90c9185..416ad48e60 100644 --- a/ext/jni/src/org/sqlite/jni/OutputPointer.java +++ b/ext/jni/src/org/sqlite/jni/OutputPointer.java @@ -36,6 +36,9 @@ package org.sqlite.jni; access to the object's value via the `value` property, whereas the JNI-level opaque types do not permit client-level code to set that property. + + Warning: do not share instances of these classes across + threads. Doing so may lead to corrupting sqlite3-internal state. */ public final class OutputPointer { diff --git a/ext/jni/src/org/sqlite/jni/Tester1.java b/ext/jni/src/org/sqlite/jni/Tester1.java index f6ca7a8547..ba32bd15c4 100644 --- a/ext/jni/src/org/sqlite/jni/Tester1.java +++ b/ext/jni/src/org/sqlite/jni/Tester1.java @@ -15,43 +15,48 @@ package org.sqlite.jni; import static org.sqlite.jni.SQLite3Jni.*; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; -public class Tester1 { +public class Tester1 implements Runnable { private static final class Metrics { int dbOpen; } - private String name; + private Integer tId; - Tester1(String name){ - this.name = name; + Tester1(Integer id){ + tId = id; } static final Metrics metrics = new Metrics(); - private static final OutputPointer.sqlite3_stmt outStmt - = new OutputPointer.sqlite3_stmt(); - public static void out(Object val){ + public synchronized static void out(Object val){ System.out.print(val); } - public static void outln(Object val){ + public synchronized static void outln(Object val){ + System.out.print(Thread.currentThread().getName()+": "); System.out.println(val); } @SuppressWarnings("unchecked") - public static void out(Object... vals){ + public synchronized static void out(Object... vals){ int n = 0; + System.out.print(Thread.currentThread().getName()+": "); for(Object v : vals) out((n++>0 ? " " : "")+v); } @SuppressWarnings("unchecked") - public static void outln(Object... vals){ + public synchronized static void outln(Object... vals){ out(vals); out("\n"); } - static int affirmCount = 0; - public static void affirm(Boolean v, String comment){ + static volatile int affirmCount = 0; + public synchronized static void affirm(Boolean v, String comment){ ++affirmCount; assert( v /* prefer assert over exception if it's enabled because the JNI layer sometimes has to suppress exceptions, @@ -66,11 +71,8 @@ public class Tester1 { private static void test1(){ outln("libversion_number:", - sqlite3_libversion_number() - + "\n" - + sqlite3_libversion() - + "\n" - + SQLITE_SOURCE_ID); + sqlite3_libversion_number(),"\n", + sqlite3_libversion(),"\n",SQLITE_SOURCE_ID); affirm(sqlite3_libversion_number() == SQLITE_VERSION_NUMBER); //outln("threadsafe = "+sqlite3_threadsafe()); affirm(SQLITE_MAX_LENGTH > 0); @@ -106,6 +108,7 @@ public class Tester1 { byte[] sqlChunk = sqlUtf8; int rc = 0; sqlite3_stmt stmt = null; + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); while(pos < sqlChunk.length){ if(pos > 0){ sqlChunk = Arrays.copyOfRange(sqlChunk, pos, @@ -144,7 +147,7 @@ public class Tester1 { } static sqlite3_stmt prepare(sqlite3 db, String sql){ - outStmt.clear(); + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); int rc = sqlite3_prepare(db, sql, outStmt); affirm( 0 == rc ); final sqlite3_stmt rv = outStmt.take(); @@ -202,11 +205,15 @@ public class Tester1 { private static void testPrepare123(){ sqlite3 db = createNewDb(); int rc; + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); rc = sqlite3_prepare(db, "CREATE TABLE t1(a);", outStmt); affirm(0 == rc); - sqlite3_stmt stmt = outStmt.get(); + sqlite3_stmt stmt = outStmt.take(); affirm(0 != stmt.getNativePointer()); rc = sqlite3_step(stmt); + if( SQLITE_DONE != rc ){ + outln("step failed ??? ",rc, sqlite3_errmsg(db)); + } affirm(SQLITE_DONE == rc); sqlite3_finalize(stmt); affirm(0 == stmt.getNativePointer()); @@ -811,6 +818,7 @@ public class Tester1 { private static void testBusy(){ final String dbName = "_busy-handler.db"; final OutputPointer.sqlite3 outDb = new OutputPointer.sqlite3(); + final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); int rc = sqlite3_open(dbName, outDb); ++metrics.dbOpen; @@ -1163,55 +1171,92 @@ public class Tester1 { testOpenDb1(); testOpenDb2(); testPrepare123(); - testBindFetchInt(); - testBindFetchInt64(); - testBindFetchDouble(); - testBindFetchText(); - testBindFetchBlob(); - testSql(); - testCollation(); - testToUtf8(); - testStatus(); - testUdf1(); - testUdfJavaObject(); - testUdfAggregate(); - testUdfWindow(); - testTrace(); - testBusy(); - testProgress(); - testCommitHook(); - testRollbackHook(); - testUpdateHook(); - testAuthorizer(); - testFts5(); - if(!fromThread){ - testAutoExtension(); + if( true ){ + testBindFetchInt(); + testBindFetchInt64(); + testBindFetchDouble(); + testBindFetchText(); + testBindFetchBlob(); + testSql(); + testCollation(); + testToUtf8(); + testStatus(); + testUdf1(); + testUdfJavaObject(); + testUdfAggregate(); + testUdfWindow(); + testTrace(); + testProgress(); + testCommitHook(); + testRollbackHook(); + testUpdateHook(); + testAuthorizer(); + if(!fromThread){ + // skip for now: messes with affirm() counts. testFts5(); + testBusy(); + testAutoExtension(); + } } } - public void run() throws Exception{ - runTests(true); + public void run(){ + try { + runTests(0!=this.tId); + }catch(Exception e){ + throw new RuntimeException(e); + }finally{ + affirm( SQLite3Jni.uncacheJniEnv() ); + affirm( !SQLite3Jni.uncacheJniEnv() ); + } } public static void main(String[] args) throws Exception { + Integer nThread = null; + boolean doSomethingForDev = false; + Integer nRepeat = 1; + for( int i = 0; i < args.length; ){ + String arg = args[i++]; + if(arg.startsWith("-")){ + arg = arg.replaceFirst("-+",""); + if(arg.equals("v")){ + doSomethingForDev = true; + //listBoundMethods(); + }else if(arg.equals("t") || arg.equals("thread")){ + nThread = Integer.parseInt(args[i++]); + }else if(arg.equals("r") || arg.equals("runs")){ + nRepeat = Integer.parseInt(args[i++]); + }else{ + throw new IllegalArgumentException("Unhandled flag:"+arg); + } + } + } - final long timeStart = System.nanoTime(); - new Tester1("main thread").runTests(false); - final long timeEnd = System.nanoTime(); - - final java.util.List liArgs = - java.util.Arrays.asList(args); - //testSleep(); - if(liArgs.indexOf("-v")>0){ - sqlite3_do_something_for_developer(); - //listBoundMethods(); + final long timeStart = System.currentTimeMillis(); + int nLoop = 0; + for( int n = 0; n < nRepeat; ++n ){ + if( nThread==null || nThread<=1 ){ + new Tester1(0).runTests(false); + }else{ + final ExecutorService ex = Executors.newFixedThreadPool( nThread ); + //final List> futures = new ArrayList<>(); + ++nLoop; + outln("Running loop #",nLoop," over ",nThread," threads."); + for( int i = 0; i < nThread; ++i ){ + ex.submit( new Tester1(i) ); + } + ex.shutdown(); + ex.awaitTermination(2, java.util.concurrent.TimeUnit.SECONDS); + ex.shutdownNow(); + } } - affirm( SQLite3Jni.uncacheJniEnv() ); - affirm( !SQLite3Jni.uncacheJniEnv() ); + final long timeEnd = System.currentTimeMillis(); outln("Tests done. Metrics:"); outln("\tAssertions checked: "+affirmCount); outln("\tDatabases opened: "+metrics.dbOpen); + if( doSomethingForDev ){ + sqlite3_do_something_for_developer(); + } int nMethods = 0; int nNatives = 0; final java.lang.reflect.Method[] declaredMethods = @@ -1232,6 +1277,6 @@ public class Tester1 { nNatives+" native methods and "+ (nMethods - nNatives)+" Java impls"); outln("\tTotal test time = " - +((timeEnd - timeStart)/1000000.0)+"ms"); + +(timeEnd - timeStart)+"ms"); } } diff --git a/ext/jni/src/org/sqlite/jni/TesterFts5.java b/ext/jni/src/org/sqlite/jni/TesterFts5.java index 6439768e29..3fd7c9bfe0 100644 --- a/ext/jni/src/org/sqlite/jni/TesterFts5.java +++ b/ext/jni/src/org/sqlite/jni/TesterFts5.java @@ -72,7 +72,7 @@ public class TesterFts5 { affirm( xDestroyCalled.value ); } - public TesterFts5(){ + public TesterFts5(boolean outputStats){ int oldAffirmCount = Tester1.affirmCount; Tester1.affirmCount = 0; final long timeStart = System.nanoTime(); diff --git a/manifest b/manifest index 0ca122a07a..0ffb551daf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\sJNI::NewStringUTF()\sfor\sthe\sremaining\scases\swhere\soutput\smay\sbe\sincompatible\swith\sMUTF-8.\sIt\sis\snow\sonly\sused\swhen\swe\sknow\sthe\soutput\sto\sbe\splain\sASCII. -D 2023-08-19T08:22:34.056 +C Add\smulti-thread\srun\smode\sto\sJNI\sTester1.\sIt\sworks\sbut\shangs\son\sexit\ssometimes\sfor\sJava\sreasons\sas\syet\snot\sunderstood. +D 2023-08-19T10:43:05.945 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -231,10 +231,10 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 -F ext/jni/GNUmakefile a9e11b92e620058558cbc1a2d49f8ec53c78d6a989b9db0b7d0b649b9f174881 +F ext/jni/GNUmakefile 28ef565d7a2df7b8db61826a4db3806e24bfc25f0bfa2f56fdd5527c93ecdb10 F ext/jni/README.md 7a614a2fa6c561205f7a53fd8626cf93a7b5711ff454fc1814517f796df398eb F ext/jni/jar-dist.make f90a553203a57934bf275bed86479485135a52f48ac5c1cfe6499ae07b0b35a4 -F ext/jni/src/c/sqlite3-jni.c a8b51e4c63572d1655dafea38b80fd63528684264c6483b5c4d1eb9098c44712 +F ext/jni/src/c/sqlite3-jni.c a4a762bff193e52a264778f64545674d5b58dbcb45478e9186d603fae2c312cd F ext/jni/src/c/sqlite3-jni.h f10d2f38720687c70ecdd5e44f6e8db98efee2caa05fc86b2d9e0c76e6cc0a18 F ext/jni/src/org/sqlite/jni/Authorizer.java 1308988f7f40579ea0e4deeaec3c6be971630566bd021c31367fe3f5140db892 F ext/jni/src/org/sqlite/jni/AutoExtension.java 18e83f6f463e306df60b2dceb65247d32af1f78af4bbbae9155411a8c6cdb093 @@ -249,14 +249,14 @@ F ext/jni/src/org/sqlite/jni/Fts5Function.java 65cde7151e441fee012250a5e03277de7 F ext/jni/src/org/sqlite/jni/Fts5PhraseIter.java 6642beda341c0b1b46af4e2d7f6f9ab03a7aede43277b2c92859176d6bce3be9 F ext/jni/src/org/sqlite/jni/Fts5Tokenizer.java 91489893596b6528c0df5cd7180bd5b55809c26e2b797fb321dfcdbc1298c060 F ext/jni/src/org/sqlite/jni/NativePointerHolder.java 9c5d901cce4f7e57c3d623f4e2476f9f79a8eed6e51b2a603f37866018e040ee -F ext/jni/src/org/sqlite/jni/OutputPointer.java d81f8bd43d2296ae373692370cfad16ddde76f5c14cd2760f7b4e1113ef56d4c +F ext/jni/src/org/sqlite/jni/OutputPointer.java 464ea85c3eba673a7b575545f69fcd8aeb398477a26d155d88cee3e2459e7802 F ext/jni/src/org/sqlite/jni/ProgressHandler.java 6f62053a828a572de809828b1ee495380677e87daa29a1c57a0e2c06b0a131dc F ext/jni/src/org/sqlite/jni/ResultCode.java ba701f20213a5f259e94cfbfdd36eb7ac7ce7797f2c6c7fca2004ff12ce20f86 F ext/jni/src/org/sqlite/jni/RollbackHook.java b04c8abcc6ade44a8a57129e33765793f69df0ba909e49ba18d73f4268d92564 F ext/jni/src/org/sqlite/jni/SQLFunction.java 09ce81c1c637e31c3a830d4c859cce95d65f5e02ff45f8bd1985b3479381bc46 F ext/jni/src/org/sqlite/jni/SQLite3Jni.java 26b3083501a9f42e9aa49b941f6b378213cf91ae1a8f705602773ed750043a3c -F ext/jni/src/org/sqlite/jni/Tester1.java 68b88b3098ce60134f4298488f890871398a77477af0a1b21797c59c911060c1 -F ext/jni/src/org/sqlite/jni/TesterFts5.java 59e22dd24af033ea8827d36225a2f3297908fb6af8818ead8850c6c6847557b1 +F ext/jni/src/org/sqlite/jni/Tester1.java 655d7109a1079be898f2631930493bd86e0c0259582014bb7af41b87d21d9a27 +F ext/jni/src/org/sqlite/jni/TesterFts5.java 3914b0a7ab0ff752c1082b1ae0c09b32827d81962fff62bcd0e13b9ec3a6f03f F ext/jni/src/org/sqlite/jni/Tracer.java a5cece9f947b0af27669b8baec300b6dd7ff859c3e6a6e4a1bd8b50f9714775d F ext/jni/src/org/sqlite/jni/UpdateHook.java e58645a1727f8a9bbe72dc072ec5b40d9f9362cb0aa24acfe93f49ff56a9016d F ext/jni/src/org/sqlite/jni/ValueHolder.java f022873abaabf64f3dd71ab0d6037c6e71cece3b8819fa10bf26a5461dc973ee @@ -2091,8 +2091,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 00a2a3736a6dcde81d920815520040f3c47f965165e7128ca1f4062e6ec7c17c -R 844cfc265e8ec15b67feebeb0f56ea87 +P 2d955eef25ab116c487ebc34c6f2d2836d310af239ef1993f5aeee5a3f68d590 +R 8a1b71ae57321d848d8875e47d990d30 U stephan -Z a99a318492abbd023ca99730c8ec8838 +Z 68dec3da5d321e77fd308ae3996624ba # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6d462b62c8..f2c6616be1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d955eef25ab116c487ebc34c6f2d2836d310af239ef1993f5aeee5a3f68d590 \ No newline at end of file +bdbaf7a4534f40e550b646979e67e7b7731566bb5a2631ed376ac85a9bec40a7 \ No newline at end of file