From: stephan Date: Tue, 22 Aug 2023 20:10:28 +0000 (+0000) Subject: More work on the JNI multi-threaded test runner. X-Git-Tag: version-3.44.0~305^2~11 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=d1c7216b2f5ac4011780d234df7566d250cf57dc;p=thirdparty%2Fsqlite.git More work on the JNI multi-threaded test runner. FossilOrigin-Name: 9a74ad716bded1e14333bf7c72392916f800d58a96240eabe4988ca5fc9e8752 --- diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c index 47269824b5..2299a4c6c0 100644 --- a/ext/jni/src/c/sqlite3-jni.c +++ b/ext/jni/src/c/sqlite3-jni.c @@ -599,8 +599,6 @@ static S3JniGlobalType S3JniGlobal = {}; /*MARKER(("Leaving PerDb mutex@%p %s.\n", env));*/ \ SJG.perDb.locker = 0; \ sqlite3_mutex_leave( SJG.perDb.mutex ) -#define MUTEX_PDB_ASSERT_LOCKED \ - assert( 0 != SJG.perDb.locker && "Misuse of S3JniGlobal.perDb.mutex" ) #define OOM_CHECK(VAR) if(!(VAR)) s3jni_oom(env) static inline void s3jni_oom(JNIEnv * const env){ @@ -903,12 +901,12 @@ static void S3JniHook_unref(JNIEnv * const env, S3JniHook * const s, int doXDest } /** - Clears s's state and moves it to the free-list. + Clears s's state and moves it to the free-list. Requires that + S3JniGlobal.perDb.mutex be unlocked. */ -static void S3JniDb_set_aside(S3JniDb * const s){ +static void S3JniDb_set_aside(JNIEnv * env, S3JniDb * const s){ if(s){ - LocalJniGetEnv; - MUTEX_PDB_ASSERT_LOCKED; + MUTEX_PDB_ENTER; //MARKER(("state@%p for db@%p setting aside\n", s, s->pDb)); assert(s->pPrev != s); assert(s->pNext != s); @@ -938,6 +936,7 @@ static void S3JniDb_set_aside(S3JniDb * const s){ SJG.perDb.aFree = s; //MARKER(("%p->pPrev@%p, pNext@%p\n", s, s->pPrev, s->pNext)); //if(s->pNext) MARKER(("next: %p->pPrev@%p\n", s->pNext, s->pNext->pPrev)); + MUTEX_PDB_LEAVE; } } @@ -1962,10 +1961,8 @@ static jint s3jni_close_db(JNIEnv * const env, jobject jDb, int version){ if(ps){ rc = 1==version ? (jint)sqlite3_close(ps->pDb) : (jint)sqlite3_close_v2(ps->pDb); if( 0==rc ){ - MUTEX_PDB_ENTER; - S3JniDb_set_aside(ps) + S3JniDb_set_aside(env, ps) /* MUST come after close() because of ps->trace. */; - MUTEX_PDB_LEAVE; NativePointerHolder_set(env, jDb, 0, &S3NphRefs.sqlite3); } } @@ -2537,9 +2534,7 @@ static int s3jni_open_post(JNIEnv * const env, S3JniEnv * const jc, assert( ps->pDb == *ppDb /* set up via s3jni_run_java_auto_extensions() */); } }else{ - MUTEX_PDB_ENTER; - S3JniDb_set_aside(ps); - MUTEX_PDB_LEAVE; + S3JniDb_set_aside(env, ps); ps = 0; } OutputPointer_set_sqlite3(env, jOut, ps ? ps->jDb : 0); @@ -3306,7 +3301,8 @@ JDECL(jbyteArray,1value_1text16be)(JENV_CSELF, jobject jpSVal){ } JDECL(void,1do_1something_1for_1developer)(JENV_CSELF){ - MARKER(("\nVarious bits of internal info:\n")); + MARKER(("\nVarious bits of internal info:\n" + "Any metrics here are invalid in multi-thread use.\n")); puts("FTS5 is " #ifdef SQLITE_ENABLE_FTS5 "available" @@ -3338,7 +3334,7 @@ JDECL(void,1do_1something_1for_1developer)(JENV_CSELF){ "\n\tenv %u" "\n\tnph inits %u" "\n\tperDb %u" - "\n\tautoExt %u container access\n", + "\n\tautoExt %u list accesses\n", SJG.metrics.nMutexEnv, SJG.metrics.nMutexEnv2, SJG.metrics.nMutexPerDb, SJG.metrics.nMutexAutoExt); printf("S3JniDb: %u alloced (*%u = %u bytes), %u recycled\n", diff --git a/ext/jni/src/org/sqlite/jni/NativePointerHolder.java b/ext/jni/src/org/sqlite/jni/NativePointerHolder.java index afe2618a00..32aee978df 100644 --- a/ext/jni/src/org/sqlite/jni/NativePointerHolder.java +++ b/ext/jni/src/org/sqlite/jni/NativePointerHolder.java @@ -28,6 +28,6 @@ package org.sqlite.jni; */ public class NativePointerHolder { //! Only set from JNI, where access permissions don't matter. - private long nativePointer = 0; + private volatile long nativePointer = 0; public final long getNativePointer(){ return nativePointer; } } diff --git a/ext/jni/src/org/sqlite/jni/Tester1.java b/ext/jni/src/org/sqlite/jni/Tester1.java index e010e24703..951ded16aa 100644 --- a/ext/jni/src/org/sqlite/jni/Tester1.java +++ b/ext/jni/src/org/sqlite/jni/Tester1.java @@ -25,7 +25,9 @@ public class Tester1 implements Runnable { //! True when running in multi-threaded mode. private static boolean mtMode = false; private static boolean takeNaps = false; - + private static boolean shuffle = false; + private static boolean listRunTests = false; + private static List testMethods = null; private static final class Metrics { int dbOpen; } @@ -159,7 +161,7 @@ public class Tester1 implements Runnable { return rv; } - private void testCompileOption(){ + private void showCompileOption(){ int i = 0; String optName; outln("compile options:"); @@ -1175,31 +1177,41 @@ public class Tester1 implements Runnable { } private void runTests(boolean fromThread) throws Exception { - if(false) testCompileOption(); + if(false) showCompileOption(); + List mlist = testMethods; + affirm( null!=mlist ); + if( shuffle ){ + mlist = new ArrayList<>( testMethods.subList(0, testMethods.size()) ); + java.util.Collections.shuffle( + mlist + //java.util.concurrent.ThreadLocalRandom.current() + ); + } + if( listRunTests ){ + synchronized(this.getClass()){ + out("Initial test"," list: "); + for(java.lang.reflect.Method m : testMethods){ + out(m.getName()+" "); + } + outln(); + + out("Running"," tests: "); + for(java.lang.reflect.Method m : mlist){ + out(m.getName()+" "); + } + outln(); + out("(That list excludes some which are hard-coded to run.)\n"); + } + } testToUtf8(); test1(); - nap(); testOpenDb1(); - nap(); testOpenDb2(); - nap(); testCollation(); - nap(); testPrepare123(); - nap(); testBindFetchInt(); - nap(); testBindFetchInt64(); - nap(); testBindFetchDouble(); - nap(); testBindFetchText(); - nap(); testBindFetchBlob(); - nap(); testSql(); - nap(); testStatus(); - nap(); testUdf1(); - nap(); testUdfJavaObject(); - nap(); testUdfAggregate(); - nap(); testUdfWindow(); - nap(); testTrace(); - nap(); testProgress(); - nap(); testCommitHook(); - nap(); testRollbackHook(); - nap(); testUpdateHook(); - nap(); testAuthorizer(); - nap(); testAutoExtension(); + int n = 0; + for(java.lang.reflect.Method m : mlist){ + ++n; + nap(); + m.invoke(this); + } + affirm( n == mlist.size() ); if(!fromThread){ testBusy(); if( !mtMode ){ @@ -1219,6 +1231,27 @@ public class Tester1 implements Runnable { } } + /** + Runs the basic sqlite3 JNI binding sanity-check suite. + + CLI flags: + + -t|-thread N: runs the tests in N threads + concurrently. Default=1. + + -r|-repeat N: repeats the tests in a loop N times, each one + consisting of the -thread value's threads. + + -shuffle: randomizes the order of most of the test functions. + + -naps: sleep small random intervals between tests in order to add + some chaos for cross-thread contention. + + -list-tests: outputs the list of tests being run, minus some + which are hard-coded, + + -v: emit some developer-mode info at the end. + */ public static void main(String[] args) throws Exception { Integer nThread = null; boolean doSomethingForDev = false; @@ -1232,8 +1265,12 @@ public class Tester1 implements Runnable { //listBoundMethods(); }else if(arg.equals("t") || arg.equals("thread")){ nThread = Integer.parseInt(args[i++]); - }else if(arg.equals("r") || arg.equals("runs")){ + }else if(arg.equals("r") || arg.equals("repeat")){ nRepeat = Integer.parseInt(args[i++]); + }else if(arg.equals("shuffle")){ + shuffle = true; + }else if(arg.equals("list-tests")){ + listRunTests = true; }else if(arg.equals("naps")){ takeNaps = true; }else{ @@ -1242,6 +1279,24 @@ public class Tester1 implements Runnable { } } + { + // Build list of tests to run from the methods named test*(). + testMethods = new ArrayList<>(); + final List excludes = new ArrayList<>(); + // Tests we want to control the order of: + excludes.add("testSleep"); + excludes.add("testToUtf8"); + excludes.add("test1"); + excludes.add("testBusy"); + excludes.add("testFts5"); + for(java.lang.reflect.Method m : Tester1.class.getDeclaredMethods()){ + final String name = m.getName(); + if( name.startsWith("test") && excludes.indexOf(name)<0 ){ + testMethods.add(m); + } + } + } + final long timeStart = System.currentTimeMillis(); int nLoop = 0; outln("libversion_number: ", diff --git a/manifest b/manifest index 71e4b52cfe..eb006ab13a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disassociate\sJNI\sdb\shandles\sfrom\sthe\sthread\sthat\screated\sthem,\sas\sit's\sno\slonger\srelevant. -D 2023-08-22T18:36:30.981 +C More\swork\son\sthe\sJNI\smulti-threaded\stest\srunner. +D 2023-08-22T20:10:28.237 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -235,7 +235,7 @@ F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a3 F ext/jni/GNUmakefile 1ccd09095447709ffd7a4f32514fd586512491c6bed06d009bab4294b451ed62 F ext/jni/README.md 975b35173debbbf3a4ab7166e14d2ffa2bacff9b6850414f09cc919805e81ba4 F ext/jni/jar-dist.make 9a03d10dbb5a74c724bfec4b76fd9e4c9865cbbc858d731cb48f38ac897d73a3 -F ext/jni/src/c/sqlite3-jni.c 50edc462e8fdf54f9b8ede692a7c865c5e4315930899276664dd6744764d4723 +F ext/jni/src/c/sqlite3-jni.c c38c18875b946a3bdc4eda0b2f19ad53b895118979ec85a630706c1c5575079b F ext/jni/src/c/sqlite3-jni.h 8b0ab1a3f0f92b75d4ff50db4a88b66a137cfb561268eb15bb3993ed174dbb74 F ext/jni/src/org/sqlite/jni/Authorizer.java 1308988f7f40579ea0e4deeaec3c6be971630566bd021c31367fe3f5140db892 F ext/jni/src/org/sqlite/jni/AutoExtension.java 3b62c915e45ce73f63343ca9195ec63592244d616a1908b7587bdd45de1b97dd @@ -249,14 +249,14 @@ F ext/jni/src/org/sqlite/jni/Fts5ExtensionApi.java 10cb2e0eb4dc5cf4241a7ccc0442a F ext/jni/src/org/sqlite/jni/Fts5Function.java 65cde7151e441fee012250a5e03277de7babcd11a0c308a832b7940574259bcc F ext/jni/src/org/sqlite/jni/Fts5PhraseIter.java 6642beda341c0b1b46af4e2d7f6f9ab03a7aede43277b2c92859176d6bce3be9 F ext/jni/src/org/sqlite/jni/Fts5Tokenizer.java 91489893596b6528c0df5cd7180bd5b55809c26e2b797fb321dfcdbc1298c060 -F ext/jni/src/org/sqlite/jni/NativePointerHolder.java 9c5d901cce4f7e57c3d623f4e2476f9f79a8eed6e51b2a603f37866018e040ee +F ext/jni/src/org/sqlite/jni/NativePointerHolder.java 8110d4cfb20884e8ed241de7420c615b040a9f9c441d9cff06f34833399244a8 F ext/jni/src/org/sqlite/jni/OutputPointer.java 464ea85c3eba673a7b575545f69fcd8aeb398477a26d155d88cee3e2459e7802 F ext/jni/src/org/sqlite/jni/ProgressHandler.java 6f62053a828a572de809828b1ee495380677e87daa29a1c57a0e2c06b0a131dc F ext/jni/src/org/sqlite/jni/ResultCode.java ba701f20213a5f259e94cfbfdd36eb7ac7ce7797f2c6c7fca2004ff12ce20f86 F ext/jni/src/org/sqlite/jni/RollbackHook.java b04c8abcc6ade44a8a57129e33765793f69df0ba909e49ba18d73f4268d92564 F ext/jni/src/org/sqlite/jni/SQLFunction.java 8c1ad92c35bcc1b2f7256cf6e229b31340ed6d1a404d487f0a9adb28ba7fc332 F ext/jni/src/org/sqlite/jni/SQLite3Jni.java 2f36370cfdec01d309720392b2c3e4af6afce0b6ece8188b5c3ed688a5a1e63a -F ext/jni/src/org/sqlite/jni/Tester1.java 58a058f718215ff32fbdf8026a2d4eb88f9d7e939a5640d5a944efafdfda4b7c +F ext/jni/src/org/sqlite/jni/Tester1.java da8bc65f52d310ae17b372eeaef25726be47d3a2052e8a33ce44606a7dc451d7 F ext/jni/src/org/sqlite/jni/TesterFts5.java c729d5b3cb91888b7e2a3a3ef450852f184697df78721574f6c0bf9043e4b84c F ext/jni/src/org/sqlite/jni/Tracer.java a5cece9f947b0af27669b8baec300b6dd7ff859c3e6a6e4a1bd8b50f9714775d F ext/jni/src/org/sqlite/jni/UpdateHook.java e58645a1727f8a9bbe72dc072ec5b40d9f9362cb0aa24acfe93f49ff56a9016d @@ -2092,8 +2092,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 02e868690f97ca728b0f2dd018aa79a9d13c85dd85b164caa895d319ae8f3ff5 -R 4fc9fbe1829e4bd8b0e17e9933291453 +P 8b78b737e66a399b04e555a8197f63a73198a4105cb2f37ffd5b0e6014302caf +R 31a03bd5ee6de6309ec09a8781691d85 U stephan -Z 9fda786e7e193bbf92304385fbc27c96 +Z 973ac484c97d56f1bb0793972fc0262a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cfa2599e0d..0f65722775 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8b78b737e66a399b04e555a8197f63a73198a4105cb2f37ffd5b0e6014302caf \ No newline at end of file +9a74ad716bded1e14333bf7c72392916f800d58a96240eabe4988ca5fc9e8752 \ No newline at end of file