From: Howard Chu Date: Mon, 20 Jun 2011 17:57:57 +0000 (-0700) Subject: More fixes, add test script X-Git-Tag: OPENLDAP_REL_ENG_2_5_0ALPHA~1329 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=05ca8c3710b3f42e3c81f8482d5c0a76bd3d8fd6;p=thirdparty%2Fopenldap.git More fixes, add test script --- diff --git a/servers/slapd/syncrepl.c b/servers/slapd/syncrepl.c index 90d719e02c..da7d2673c1 100644 --- a/servers/slapd/syncrepl.c +++ b/servers/slapd/syncrepl.c @@ -1783,28 +1783,31 @@ typedef struct OpExtraSync { /* Copy the original modlist, split Replace ops into Delete/Add, * and drop mod opattrs since this modification is in the past. */ -static Modifications *mods_dup( Operation *op, Modifications *modlist ) +static Modifications *mods_dup( Operation *op, Modifications *modlist, int match ) { Modifications *mod, *modnew = NULL, *modtail = NULL; int size; for ( ; modlist; modlist = modlist->sml_next ) { - if ( modlist->sml_desc == slap_schema.si_ad_modifiersName || - modlist->sml_desc == slap_schema.si_ad_modifyTimestamp || - modlist->sml_desc == slap_schema.si_ad_entryCSN ) - continue; - if ( modlist->sml_op == LDAP_MOD_REPLACE ) { - mod = op->o_tmpalloc( sizeof(Modifications), op->o_tmpmemctx ); - mod->sml_desc = modlist->sml_desc; - mod->sml_values = NULL; - mod->sml_nvalues = NULL; - mod->sml_op = LDAP_MOD_DELETE; - mod->sml_numvals = 0; - mod->sml_flags = 0; - if ( !modnew ) - modnew = mod; - if ( modtail ) - modtail->sml_next = mod; - modtail = mod; + /* older ops */ + if ( match < 0 ) { + if ( modlist->sml_desc == slap_schema.si_ad_modifiersName || + modlist->sml_desc == slap_schema.si_ad_modifyTimestamp || + modlist->sml_desc == slap_schema.si_ad_entryCSN ) + continue; + if ( modlist->sml_op == LDAP_MOD_REPLACE ) { + mod = op->o_tmpalloc( sizeof(Modifications), op->o_tmpmemctx ); + mod->sml_desc = modlist->sml_desc; + mod->sml_values = NULL; + mod->sml_nvalues = NULL; + mod->sml_op = LDAP_MOD_DELETE; + mod->sml_numvals = 0; + mod->sml_flags = 0; + if ( !modnew ) + modnew = mod; + if ( modtail ) + modtail->sml_next = mod; + modtail = mod; + } } if ( modlist->sml_numvals ) { size = (modlist->sml_numvals+1) * sizeof(struct berval); @@ -1827,13 +1830,15 @@ static Modifications *mods_dup( Operation *op, Modifications *modlist ) mod->sml_values = (BerVarray)(mod+1); for (i=0; isml_numvals; i++) mod->sml_values[i] = modlist->sml_values[i]; + BER_BVZERO(&mod->sml_values[i]); if ( modlist->sml_nvalues ) { mod->sml_nvalues = mod->sml_values + mod->sml_numvals + 1; for (i=0; isml_numvals; i++) mod->sml_nvalues[i] = modlist->sml_nvalues[i]; + BER_BVZERO(&mod->sml_nvalues[i]); } } - if ( modlist->sml_op == LDAP_MOD_REPLACE ) + if ( match < 0 && modlist->sml_op == LDAP_MOD_REPLACE ) mod->sml_op = LDAP_MOD_ADD; else mod->sml_op = modlist->sml_op; @@ -1869,6 +1874,10 @@ compare_vals( Modifications *m1, Modifications *m2 ) if ( m1->sml_nvalues ) m1->sml_nvalues[k] = m1->sml_nvalues[k+1]; } + BER_BVZERO(&m1->sml_values[k]); + if ( m1->sml_nvalues ) { + BER_BVZERO(&m1->sml_nvalues[k]); + } m1->sml_numvals--; i--; } @@ -1955,15 +1964,14 @@ syncrepl_modify_cb( Operation *op, SlapReply *rs ) modify_ctxt *mx = sc->sc_private; Modifications *ml; - op->o_callback = sc->sc_next; - op->o_tmpfree( sc, op->o_tmpmemctx ); - op->orm_no_opattrs = 0; op->orm_modlist = mx->mx_orig; for ( ml = mx->mx_free; ml; ml = mx->mx_free ) { mx->mx_free = ml->sml_next; op->o_tmpfree( ml, op->o_tmpmemctx ); } + op->o_callback = sc->sc_next; + op->o_tmpfree( sc, op->o_tmpmemctx ); return SLAP_CB_CONTINUE; } @@ -2010,14 +2018,6 @@ syncrepl_op_modify( Operation *op, SlapReply *rs ) &mod->sml_nvalues[0], &a->a_nvals[0], &text ); overlay_entry_release_ov( op, e, 0, on ); } - /* mod is newer, let it go */ - if ( match > 0 ) { - for ( mod = op->orm_modlist; mod; mod=mod->sml_next ) { - if ( mod->sml_op == LDAP_MOD_DELETE ) - mod->sml_op = SLAP_MOD_SOFTDEL; - } - return SLAP_CB_CONTINUE; - } /* equal? Should never happen */ if ( match == 0 ) return LDAP_SUCCESS; @@ -2043,20 +2043,28 @@ syncrepl_op_modify( Operation *op, SlapReply *rs ) * * 4. Swap original modlist back in response callback so * that accesslog logs the original mod. + * + * Even if the mod is newer, other out-of-order changes may + * have been committed, forcing us to tweak the modlist: + * 1. Save/copy original modlist. + * 2. Change deletes to soft deletes. + * 3. Change Adds of single-valued attrs to Replace. */ - newlist = mods_dup( op, op->orm_modlist ); + newlist = mods_dup( op, op->orm_modlist, match ); - { + /* mod is older */ + if ( match < 0 ) { Operation op2 = *op; AttributeName an[2]; const char *text; - slap_callback cb; struct berval bv; char *ptr; + Modifications *ml; int size, rc; SlapReply rs1 = {0}; resolve_ctxt rx = { si, newlist }; + slap_callback cb = { NULL, syncrepl_resolve_cb, NULL, &rx }; op2.o_tag = LDAP_REQ_SEARCH; op2.ors_scope = LDAP_SCOPE_SUBTREE; @@ -2083,8 +2091,6 @@ syncrepl_op_modify( Operation *op, SlapReply *rs ) op2.ors_filter = str2filter_x( op, op2.ors_filterstr.bv_val ); op2.o_callback = &cb; - cb.sc_response = syncrepl_resolve_cb; - cb.sc_private = ℞ op2.o_bd = select_backend( &op2.o_req_ndn, 1 ); op2.o_bd->be_search( &op2, &rs1 ); newlist = rx.rx_mods; @@ -2099,6 +2105,7 @@ syncrepl_op_modify( Operation *op, SlapReply *rs ) sc->sc_response = syncrepl_modify_cb; sc->sc_private = mx; sc->sc_next = op->o_callback; + sc->sc_cleanup = NULL; op->o_callback = sc; op->orm_no_opattrs = 1; mx->mx_orig = op->orm_modlist; @@ -2108,6 +2115,11 @@ syncrepl_op_modify( Operation *op, SlapReply *rs ) ml->sml_flags = 0; ml->sml_op = SLAP_MOD_SOFTDEL; } + else if ( ml->sml_op == LDAP_MOD_DELETE ) + ml->sml_op = SLAP_MOD_SOFTDEL; + else if ( ml->sml_op == LDAP_MOD_ADD && + ml->sml_desc->ad_type->sat_atype.at_single_value ) + ml->sml_op = LDAP_MOD_REPLACE; } op->orm_modlist = newlist; op->o_csn = mod->sml_nvalues[0]; @@ -2286,6 +2298,7 @@ syncrepl_message_to_op( rc = op->o_bd->be_modify( op, &rs ); if ( SLAP_MULTIMASTER( op->o_bd )) { LDAP_SLIST_REMOVE( &op->o_extra, &oes.oe, OpExtra, oe_next ); + BER_BVZERO( &op->o_csn ); } modlist = op->orm_modlist; Debug( rc ? LDAP_DEBUG_ANY : LDAP_DEBUG_SYNC, diff --git a/tests/scripts/test063-delta-multimaster b/tests/scripts/test063-delta-multimaster new file mode 100755 index 0000000000..af043057dd --- /dev/null +++ b/tests/scripts/test063-delta-multimaster @@ -0,0 +1,529 @@ +#! /bin/sh +# $OpenLDAP$ +## This work is part of OpenLDAP Software . +## +## Copyright 1998-2011 The OpenLDAP Foundation. +## All rights reserved. +## +## Redistribution and use in source and binary forms, with or without +## modification, are permitted only as authorized by the OpenLDAP +## Public License. +## +## A copy of this license is available in the file LICENSE in the +## top-level directory of the distribution or, alternatively, at +## . + +echo "running defines.sh" +. $SRCDIR/scripts/defines.sh + +if test $SYNCPROV = syncprovno; then + echo "Syncrepl provider overlay not available, test skipped" + exit 0 +fi +if test $ACCESSLOG = accesslogno; then + echo "Accesslog overlay not available, test skipped" + exit 0 +fi + +MMR=2 + +XDIR=$TESTDIR/srv +TMP=$TESTDIR/tmp + +mkdir -p $TESTDIR + +$SLAPPASSWD -g -n >$CONFIGPWF + +if test x"$SYNCMODE" = x ; then + SYNCMODE=rp +fi +case "$SYNCMODE" in + ro) + SYNCTYPE="type=refreshOnly interval=00:00:00:03" + ;; + rp) + SYNCTYPE="type=refreshAndPersist" + ;; + *) + echo "unknown sync mode $SYNCMODE" + exit 1; + ;; +esac + +# +# Test delta-sync mmr +# - start servers +# - configure over ldap +# - populate over ldap +# - configure syncrepl over ldap +# - break replication +# - modify each server separately +# - restore replication +# - compare results +# + +nullExclude="" +test $BACKEND = null && nullExclude="# " + +KILLPIDS= + +echo "Initializing server configurations..." +n=1 +while [ $n -le $MMR ]; do + +DBDIR=${XDIR}$n/db +CFDIR=${XDIR}$n/slapd.d + +mkdir -p ${XDIR}$n $DBDIR.1 $DBDIR.2 $CFDIR + +o=`expr 3 - $n` +cat > $TMP <> $TMP +dn: cn=module,cn=config +objectClass: olcModuleList +cn: module +olcModulePath: $TESTWD/../servers/slapd/overlays +EOF + if [ "$SYNCPROV" = syncprovmod ]; then + echo "olcModuleLoad: syncprov.la" >> $TMP + fi + if [ "$ACCESSLOG" = accesslogmod ]; then + echo "olcModuleLoad: accesslog.la" >> $TMP + fi + echo "" >> $TMP +fi + +if [ "$BACKENDTYPE" = mod ]; then +cat <> $TMP +dn: cn=module,cn=config +objectClass: olcModuleList +cn: module +olcModulePath: $TESTWD/../servers/slapd/back-$BACKEND +olcModuleLoad: back_$BACKEND.la + +EOF +fi +MYURI=`eval echo '$URI'$n` +PROVIDERURI=`eval echo '$URI'$o` +if [ $BACKEND = hdb -o $BACKEND = bdb ]; then +INDEX1="olcDbIndex: objectClass,entryCSN,reqStart,reqDN eq" +INDEX2="olcDbIndex: objectClass,entryCSN,entryUUID eq" +else +INDEX1= +INDEX2= +fi +cat >> $TMP < $TESTOUT 2>&1 +PORT=`eval echo '$PORT'$n` +echo "Starting server $n on TCP/IP port $PORT..." +cd ${XDIR}${n} +LOG=`eval echo '$LOG'$n` +$SLAPD -F slapd.d -h $MYURI -d $LVL $TIMING > $LOG 2>&1 & +PID=$! +if test $WAIT != 0 ; then + echo PID $PID + read foo +fi +KILLPIDS="$PID $KILLPIDS" +cd $TESTWD + +echo "Using ldapsearch to check that server $n is running..." +for i in 0 1 2 3 4 5; do + $LDAPSEARCH -s base -b "" -H $MYURI \ + 'objectclass=*' > /dev/null 2>&1 + RC=$? + if test $RC = 0 ; then + break + fi + echo "Waiting 5 seconds for slapd to start..." + sleep 5 +done + +if test $RC != 0 ; then + echo "ldapsearch failed ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +if [ $n = 1 ]; then +echo "Using ldapadd for context on server 1..." +$LDAPADD -D "$MANAGERDN" -H $URI1 -w $PASSWD -f $LDIFORDEREDCP \ + >> $TESTOUT 2>&1 +RC=$? +if test $RC != 0 ; then + echo "ldapadd failed for server $n database ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi +fi + +n=`expr $n + 1` +done + +echo "Using ldapadd to populate server 1..." +$LDAPADD -D "$MANAGERDN" -H $URI1 -w $PASSWD -f $LDIFORDEREDNOCP \ + >> $TESTOUT 2>&1 +RC=$? +if test $RC != 0 ; then + echo "ldapadd failed for server $n database ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..." +sleep $SLEEP1 + +n=1 +while [ $n -le $MMR ]; do +PORT=`expr $BASEPORT + $n` +URI="ldap://${LOCALHOST}:$PORT/" + +echo "Using ldapsearch to read all the entries from server $n..." +$LDAPSEARCH -S "" -b "$BASEDN" -D "$MANAGERDN" -H $URI -w $PASSWD \ + 'objectclass=*' > $TESTDIR/server$n.out 2>&1 +RC=$? + +if test $RC != 0 ; then + echo "ldapsearch failed at server $n ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi +$LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt +n=`expr $n + 1` +done + +n=2 +while [ $n -le $MMR ]; do +echo "Comparing retrieved entries from server 1 and server $n..." +$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT + +if test $? != 0 ; then + echo "test failed - server 1 and server $n databases differ" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit 1 +fi +n=`expr $n + 1` +done + +echo "Using ldapadd to populate server 2..." +$LDAPADD -D "$MANAGERDN" -H $URI2 -w $PASSWD -f $LDIFADD1 \ + >> $TESTOUT 2>&1 +RC=$? +if test $RC != 0 ; then + echo "ldapadd failed for server 2 database ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..." +sleep $SLEEP1 + +n=1 +while [ $n -le $MMR ]; do +PORT=`expr $BASEPORT + $n` +URI="ldap://${LOCALHOST}:$PORT/" + +echo "Using ldapsearch to read all the entries from server $n..." +$LDAPSEARCH -S "" -b "$BASEDN" -D "$MANAGERDN" -H $URI -w $PASSWD \ + 'objectclass=*' > $TESTDIR/server$n.out 2>&1 +RC=$? + +if test $RC != 0 ; then + echo "ldapsearch failed at server $n ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi +$LDIFFILTER < $TESTDIR/server$n.out > $TESTDIR/server$n.flt +n=`expr $n + 1` +done + +n=2 +while [ $n -le $MMR ]; do +echo "Comparing retrieved entries from server 1 and server $n..." +$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT + +if test $? != 0 ; then + echo "test failed - server 1 and server $n databases differ" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit 1 +fi +n=`expr $n + 1` +done + +echo "Breaking replication between server 1 and 2..." +n=1 +while [ $n -le $MMR ]; do +o=`expr 3 - $n` +MYURI=`eval echo '$URI'$n` +PROVIDERURI=`eval echo '$URI'$o` +$LDAPMODIFY -D cn=config -H $MYURI -y $CONFIGPWF > $TESTOUT 2>&1 <> $TESTOUT 2>&1 << EOF +dn: $THEDN +changetype: modify +add: description +description: Amazing + +EOF +RC=$? +if test $RC != 0 ; then + echo "ldapmodify failed for server 1 database ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +$LDAPMODIFY -D "$MANAGERDN" -H $URI2 -w $PASSWD \ + >> $TESTOUT 2>&1 << EOF +dn: $THEDN +changetype: modify +add: description +description: Stupendous + +EOF +RC=$? +if test $RC != 0 ; then + echo "ldapmodify failed for server 2 database ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +$LDAPMODIFY -D "$MANAGERDN" -H $URI1 -w $PASSWD \ + >> $TESTOUT 2>&1 << EOF +dn: $THEDN +changetype: modify +delete: description +description: Outstanding +- +add: description +description: Mindboggling + +EOF +RC=$? +if test $RC != 0 ; then + echo "ldapmodify failed for server 1 database ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +$LDAPMODIFY -D "$MANAGERDN" -H $URI2 -w $PASSWD \ + >> $TESTOUT 2>&1 << EOF +dn: $THEDN +changetype: modify +delete: description +description: OutStanding +- +add: description +description: Bizarre + +EOF +RC=$? +if test $RC != 0 ; then + echo "ldapmodify failed for server 2 database ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +$LDAPMODIFY -D "$MANAGERDN" -H $URI1 -w $PASSWD \ + >> $TESTOUT 2>&1 << EOF +dn: $THEDN +changetype: modify +add: carLicense +carLicense: 123-XYZ +- +add: employeeNumber +employeeNumber: 32 + +EOF +RC=$? +if test $RC != 0 ; then + echo "ldapmodify failed for server 1 database ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +$LDAPMODIFY -D "$MANAGERDN" -H $URI2 -w $PASSWD \ + >> $TESTOUT 2>&1 << EOF +dn: $THEDN +changetype: modify +add: employeeType +employeeType: deadwood +- +add: employeeNumber +employeeNumber: 64 + +EOF +RC=$? +if test $RC != 0 ; then + echo "ldapmodify failed for server 2 database ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Restoring replication between server 1 and 2..." +n=1 +while [ $n -le $MMR ]; do +o=`expr 3 - $n` +MYURI=`eval echo '$URI'$n` +PROVIDERURI=`eval echo '$URI'$o` +$LDAPMODIFY -D cn=config -H $MYURI -y $CONFIGPWF > $TESTOUT 2>&1 < $TESTDIR/server$n.out 2>&1 +RC=$? + +if test $RC != 0 ; then + echo "ldapsearch failed at server $n ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi +$LDIFFILTER -s $BACKEND=a < $TESTDIR/server$n.out > $TESTDIR/server$n.flt +n=`expr $n + 1` +done + +n=2 +while [ $n -le $MMR ]; do +echo "Comparing retrieved entries from server 1 and server $n..." +$CMP $MASTERFLT $TESTDIR/server$n.flt > $CMPOUT + +if test $? != 0 ; then + echo "test failed - server 1 and server $n databases differ" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit 1 +fi +n=`expr $n + 1` +done + +test $KILLSERVERS != no && kill -HUP $KILLPIDS + +echo ">>>>> Test succeeded" + +test $KILLSERVERS != no && wait + +exit 0