From: Howard Chu Date: Thu, 28 Feb 2019 13:24:13 +0000 (+0000) Subject: Add persist support X-Git-Tag: OPENLDAP_REL_ENG_2_5_0ALPHA~67^2~90 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=47102fccedbd3c10f3c348044fa00c41e793efc1;p=thirdparty%2Fopenldap.git Add persist support --- diff --git a/servers/slapd/syncrepl.c b/servers/slapd/syncrepl.c index 826c056392..f19217f0ac 100644 --- a/servers/slapd/syncrepl.c +++ b/servers/slapd/syncrepl.c @@ -428,6 +428,7 @@ init_syncrepl(syncinfo_t *si) for ( i = 0; dsee_descs[ i ] != NULL; i++ ) { attrs[ n++ ] = ch_strdup ( dsee_descs[i]->ad_cname.bv_val ); } + attrs[n] = NULL; } si->si_attrs = attrs; @@ -541,7 +542,7 @@ ldap_sync_search( } ldap_msgfree( res ); if ( gotfirst && gotlast ) { - if ( !si->si_lastchange || si->si_lastchange < first ) + if ( si->si_lastchange < first || (!si->si_lastchange && !si->si_refreshDone )) si->si_logstate = SYNCLOG_FALLBACK; /* if we're in logging mode, it will update si_lastchange itself */ if ( si->si_logstate == SYNCLOG_FALLBACK ) @@ -628,7 +629,16 @@ ldap_sync_search( #endif #ifdef DO_DSEE if ( si->si_syncdata == SYNCDATA_CHANGELOG ) { - ctrls[0] = NULL; + if ( si->si_logstate == SYNCLOG_LOGGING && si->si_type == LDAP_SYNC_REFRESH_AND_PERSIST ) { + c[0].ldctl_oid = LDAP_CONTROL_PERSIST_REQUEST; + c[0].ldctl_iscritical = 0; + rc = ldap_create_persistentsearch_control_value( si->si_ld, LDAP_CONTROL_PERSIST_ENTRY_CHANGE_ADD, + 0, 1, &c[0].ldctl_value ); + ctrls[0] = &c[0]; + ctrls[1] = NULL; + } else { + ctrls[0] = NULL; + } } else #endif { @@ -1090,12 +1100,30 @@ do_syncrep2( break; } #endif + ldap_get_entry_controls( si->si_ld, msg, &rctrls ); + ldap_get_dn_ber( si->si_ld, msg, NULL, &bdn ); + if (!bdn.bv_len) { + bdn.bv_val = empty; + bdn.bv_len = sizeof(empty)-1; + } #ifdef DO_DSEE if ( si->si_syncdata == SYNCDATA_CHANGELOG ) { if ( si->si_logstate == SYNCLOG_LOGGING ) { rc = syncrepl_message_to_op( si, op, msg ); if ( rc ) - si->si_logstate = SYNCLOG_FALLBACK; + goto logerr; + if ( si->si_type == LDAP_SYNC_REFRESH_AND_PERSIST && rctrls ) { + LDAPControl **next = NULL; + /* The notification control is only sent during persist phase */ + rctrlp = ldap_control_find( LDAP_CONTROL_PERSIST_ENTRY_CHANGE_NOTICE, rctrls, &next ); + if ( rctrlp ) { + if ( !si->si_refreshDone ) + si->si_refreshDone = 1; + if ( si->si_refreshDone ) + syncrepl_dsee_update( si, op ); + } + } + } else { syncstate = DSEE_SYNC_ADD; rc = syncrepl_message_to_entry( si, op, msg, @@ -1111,12 +1139,6 @@ do_syncrep2( break; } #endif - ldap_get_entry_controls( si->si_ld, msg, &rctrls ); - ldap_get_dn_ber( si->si_ld, msg, NULL, &bdn ); - if (!bdn.bv_len) { - bdn.bv_val = empty; - bdn.bv_len = sizeof(empty)-1; - } /* we can't work without the control */ if ( rctrls ) { LDAPControl **next = NULL; @@ -1269,7 +1291,9 @@ do_syncrep2( syncCookie.ctxcsn ) { rc = syncrepl_updateCookie( si, op, &syncCookie, 0 ); - } else switch ( rc ) { + } else +logerr: + switch ( rc ) { case LDAP_ALREADY_EXISTS: case LDAP_NO_SUCH_OBJECT: case LDAP_NO_SUCH_ATTRIBUTE: @@ -1365,8 +1389,13 @@ do_syncrep2( if ( si->si_syncdata == SYNCDATA_CHANGELOG && err == LDAP_SUCCESS ) { rc = syncrepl_dsee_update( si, op ); if ( rc == LDAP_SUCCESS ) { - rc = -2; /* schedule a re-poll */ - si->si_logstate = SYNCLOG_LOGGING; + if ( si->si_logstate == SYNCLOG_FALLBACK ) { + si->si_logstate = SYNCLOG_LOGGING; + si->si_refreshDone = 1; + rc = LDAP_SYNC_REFRESH_REQUIRED; + } else { + rc = -2; + } } goto done; } diff --git a/tests/data/slapd-dsee-slave2.conf b/tests/data/slapd-dsee-slave2.conf new file mode 100644 index 0000000000..7195c2ddce --- /dev/null +++ b/tests/data/slapd-dsee-slave2.conf @@ -0,0 +1,63 @@ +# slave slapd config -- for testing of SYNC replication +# $OpenLDAP$ +## This work is part of OpenLDAP Software . +## +## Copyright 1998-2018 The OpenLDAP Foundation. +## All rights reserved. +## +## Redistribution and use in source and binary forms, with or without +## modification, are permitted only as authorized by the OpenLDAP +## Public License. +## +## A copy of this license is available in the file LICENSE in the +## top-level directory of the distribution or, alternatively, at +## . + +include @SCHEMADIR@/core.schema +include @SCHEMADIR@/cosine.schema +include @SCHEMADIR@/inetorgperson.schema +include @SCHEMADIR@/openldap.schema +include @SCHEMADIR@/nis.schema +include @SCHEMADIR@/dsee.schema +# +pidfile @TESTDIR@/slapd.4.pid +argsfile @TESTDIR@/slapd.4.args + +#mod#modulepath ../servers/slapd/back-@BACKEND@/ +#mod#moduleload back_@BACKEND@.la +#monitormod#modulepath ../servers/slapd/back-monitor/ +#monitormod#moduleload back_monitor.la + +####################################################################### +# consumer database definitions +####################################################################### + +database @BACKEND@ +suffix "dc=example,dc=com" +rootdn "cn=Replica,dc=example,dc=com" +rootpw secret +#null#bind on +#~null~#directory @TESTDIR@/db.4.a +#indexdb#index objectClass eq +#indexdb#index cn,sn,uid pres,eq,sub +#indexdb#index entryUUID,entryCSN eq +#ndb#dbname db_4 +#ndb#include @DATADIR@/ndb.conf + +# Don't change syncrepl spec yet +syncrepl rid=1 + provider=@URI1@ + binddn="cn=Directory Manager" + bindmethod=simple + credentials=secret21 + searchbase="dc=example,dc=com" + filter="(objectClass=*)" + schemachecking=off + scope=sub + type=refreshAndPersist + logbase="cn=changelog" + syncdata=changelog + retry="3 5 300 5" +updateref @URI1@ + +#monitor#database monitor diff --git a/tests/data/test-dirsync-nocp.ldif b/tests/data/test-dirsync-nocp.ldif index daeda7979a..757c4ed161 100644 --- a/tests/data/test-dirsync-nocp.ldif +++ b/tests/data/test-dirsync-nocp.ldif @@ -237,8 +237,8 @@ cn: ITD Staff objectclass: groupOfUniqueNames objectclass: domainRelatedObject uniquemember: cn=Manager,dc=example,dc=com -uniquemember: cn=Bjorn Jensen,OU=Information Technology Division,ou=People,dc=example,dc=com -uniquemember: cn=James A Jones 2,ou=Information Technology Division,ou=People,dc=example,dc=com +uniquemember: cn=Bjorn Jensen,OU=Information Technology Division,ou=People,dc=example,dc=com +uniquemember: cn=James A Jones 2,ou=Information Technology Division,ou=People,dc=example,dc=com uniquemember: cn=John Doe,ou=Information Technology Division,ou=People,dc=example,dc=com associatedDomain: test.openldap.org diff --git a/tests/scripts/defines.sh b/tests/scripts/defines.sh index a52dc285df..33dc37022c 100755 --- a/tests/scripts/defines.sh +++ b/tests/scripts/defines.sh @@ -108,6 +108,7 @@ P2SRSLAVECONF=$DATADIR/slapd-syncrepl-slave-persist2.conf P3SRSLAVECONF=$DATADIR/slapd-syncrepl-slave-persist3.conf DIRSYNC1CONF=$DATADIR/slapd-dirsync1.conf DSEESYNC1CONF=$DATADIR/slapd-dsee-slave1.conf +DSEESYNC2CONF=$DATADIR/slapd-dsee-slave2.conf REFSLAVECONF=$DATADIR/slapd-ref-slave.conf SCHEMACONF=$DATADIR/slapd-schema.conf TLSCONF=$DATADIR/slapd-tls.conf diff --git a/tests/scripts/test075-dsee-persist b/tests/scripts/test075-dsee-persist new file mode 100755 index 0000000000..359624fb42 --- /dev/null +++ b/tests/scripts/test075-dsee-persist @@ -0,0 +1,420 @@ +#! /bin/sh +# $OpenLDAP$ +## This work is part of OpenLDAP Software . +## +## Copyright 1998-2019 The OpenLDAP Foundation. +## All rights reserved. +## +## Redistribution and use in source and binary forms, with or without +## modification, are permitted only as authorized by the OpenLDAP +## Public License. +## +## A copy of this license is available in the file LICENSE in the +## top-level directory of the distribution or, alternatively, at +## . + +echo "running defines.sh" +. $SRCDIR/scripts/defines.sh + +OPATTRS="creatorsName createTimestamp modifiersName modifyTimestamp" + +if test -z `which dsadm`; then + echo "DSEE dsadm not in path, test skipped" + exit 0 +fi + +mkdir -p $TESTDIR $DBDIR4 + +# +# Test replication: +# - start provider +# - start consumer +# - populate over ldap +# - perform some modifies and deleted +# - attempt to modify the consumer (referral or chain) +# - retrieve database over ldap and compare against expected results +# + +DSEEPW=secret21 +DSEEDN="cn=Directory Manager" +DSEEPWF=$TESTDIR/dseepw + +echo "secret21" > $DSEEPWF + +echo "Setting up DSEE provider slapd on TCP/IP port $PORT1..." +dsadm create -p $PORT1 -w $DSEEPWF $DBDIR1 +dsadm start $DBDIR1 +dsconf create-suffix -c -p $PORT1 -w $DSEEPWF $BASEDN +dsconf set-server-prop -p $PORT1 -w $DSEEPWF moddn-enabled:on +dsconf set-server-prop -p $PORT1 -w $DSEEPWF retro-cl-enabled:on +dsadm restart $DBDIR1 +PID=`basename $DBDIR1/locks/server/*` +KILLPIDS="$PID" + +sleep 1 + +echo "Using ldapsearch to check that provider slapd is running..." +for i in 0 1 2 3 4 5; do + $LDAPSEARCH -s base -b "$BASEDN" -h $LOCALHOST -p $PORT1 \ + 'objectclass=*' > /dev/null 2>&1 + RC=$? + if test $RC = 0 ; then + break + fi + echo "Waiting 5 seconds for slapd to start..." + sleep 5 +done + +if test $RC != 0 ; then + echo "ldapsearch failed ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Starting consumer slapd on TCP/IP port $PORT4..." +. $CONFFILTER $BACKEND $MONITORDB < $DSEESYNC2CONF > $CONF4 +$SLAPD -f $CONF4 -h $URI4 -d $LVL $TIMING > $LOG4 2>&1 & +SLAVEPID=$! +if test $WAIT != 0 ; then + echo SLAVEPID $SLAVEPID + read foo +fi +KILLPIDS="$KILLPIDS $SLAVEPID" + +sleep 1 + +echo "Using ldapsearch to check that consumer slapd is running..." +for i in 0 1 2 3 4 5; do + $LDAPSEARCH -s base -b "$MONITOR" -h $LOCALHOST -p $PORT4 \ + 'objectclass=*' > /dev/null 2>&1 + RC=$? + if test $RC = 0 ; then + break + fi + echo "Waiting 5 seconds for slapd to start..." + sleep 5 +done + +if test $RC != 0 ; then + echo "ldapsearch failed ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +# using LDIFDIRSYNCNOCP to avoid custom OpenLDAP schema +echo "Using ldapadd to populate the provider directory..." +$LDAPADD -D "$DSEEDN" -h $LOCALHOST -p $PORT1 -w $DSEEPW < \ + $LDIFDIRSYNCNOCP > /dev/null 2>&1 +RC=$? +if test $RC != 0 ; then + echo "ldapadd failed ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..." +sleep $SLEEP1 + +echo "Using ldapsearch to read all the entries from the provider..." +$LDAPSEARCH -S "" -b "$BASEDN" -h $LOCALHOST -p $PORT1 \ + -D "$DSEEDN" -w "$DSEEPW" \ + '(objectclass=*)' '*' $OPATTRS > $MASTEROUT 2>&1 +RC=$? + +if test $RC != 0 ; then + echo "ldapsearch failed at provider ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Using ldapsearch to read all the entries from the consumer..." +$LDAPSEARCH -S "" -b "$BASEDN" -h $LOCALHOST -p $PORT4 \ + '(objectclass=*)' '*' $OPATTRS > $SLAVEOUT 2>&1 +RC=$? + +if test $RC != 0 ; then + echo "ldapsearch failed at consumer ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Filtering provider results..." +$LDIFFILTER -s a < $MASTEROUT > $MASTERFLT +echo "Filtering consumer results..." +$LDIFFILTER -s a < $SLAVEOUT > $SLAVEFLT + +echo "Comparing retrieved entries from provider and consumer..." +$CMP $MASTERFLT $SLAVEFLT > $CMPOUT + +if test $? != 0 ; then + echo "test failed - provider and consumer databases differ" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit 1 +fi + +echo "Stopping the provider, sleeping 10 seconds and restarting it..." +kill -HUP "$PID" +wait $PID +sleep 10 +echo "RESTART" >> $LOG1 +dsadm start $DBDIR1 +PID=`basename $DBDIR1/locks/server/*` +if test $WAIT != 0 ; then + echo PID $PID + read foo +fi +KILLPIDS="$PID $SLAVEPID" + +sleep 1 + +echo "Using ldapsearch to check that provider slapd is running..." +for i in 0 1 2 3 4 5; do + $LDAPSEARCH -s base -b "$BASEDN" -h $LOCALHOST -p $PORT1 \ + 'objectclass=*' > /dev/null 2>&1 + RC=$? + if test $RC = 0 ; then + break + fi + echo "Waiting 5 seconds for slapd to start..." + sleep 5 +done + +echo "Waiting $SLEEP1 seconds for consumer to reconnect..." +sleep $SLEEP1 + +if test $RC != 0 ; then + echo "ldapsearch failed ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Using ldapmodify to modify provider directory..." + +# +# Do some modifications +# + +$LDAPMODIFY -v -D "$DSEEDN" -h $LOCALHOST -p $PORT1 -w $DSEEPW > \ + $TESTOUT 2>&1 << EOMODS +dn: cn=James A Jones 1, ou=Alumni Association, ou=People, dc=example,dc=com +changetype: modify +add: carLicense +carLicense: Orange Juice +- +delete: sn +sn: Jones +- +add: sn +sn: Jones + +dn: cn=Bjorn Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com +changetype: modify +replace: carLicense +carLicense: Iced Tea + +dn: cn=ITD Staff,ou=Groups,dc=example,dc=com +changetype: modify +delete: uniquemember +uniquemember: cn=James A Jones 2, ou=Information Technology Division, ou=People, dc=example,dc=com +uniquemember: cn=Bjorn Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com +- +add: uniquemember +uniquemember: cn=Dorothy Stevens,ou=Alumni Association,ou=People,dc=example,dc=com +uniquemember: cn=James A Jones 1,ou=Alumni Association,ou=People,dc=example,dc=com + +dn: cn=All Staff,ou=Groups,dc=example,dc=com +changetype: modify +delete: description + +dn: cn=Gern Jensen,ou=Information Technology Division,ou=People,dc=example,dc=com +changetype: add +objectclass: inetOrgPerson +cn: Gern Jensen +sn: Jensen +uid: gjensen +title: Chief Investigator, ITD +postaladdress: ITD $ 535 W. William St $ Ann Arbor, MI 48103 +seealso: cn=All Staff,ou=Groups,dc=example,dc=com +carLicense: Coffee +homepostaladdress: 844 Brown St. Apt. 4 $ Ann Arbor, MI 48104 +description: Very odd +facsimiletelephonenumber: +1 313 555 7557 +facsimiletelephonenumber: +1 313 555 9998 +facsimiletelephonenumber: +1 313 555 9999 +telephonenumber: +1 313 555 8343 +mail: gjensen@mailgw.example.com +homephone: +1 313 555 8844 + +# modify attribute with no matching rule (ITS#6458) +dn: cn=Gern Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com +changetype: modify +replace: facsimiletelephonenumber +facsimiletelephonenumber: +1 313 555 9998 +facsimiletelephonenumber: +1 313 555 9999 + +dn: cn=Gern Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com +changetype: modify +replace: facsimiletelephonenumber +facsimiletelephonenumber: +1 313 555 9998 +facsimiletelephonenumber: +1 313 555 9999 +facsimiletelephonenumber: +1 313 555 7557 + +dn: cn=Gern Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com +changetype: modify +replace: facsimiletelephonenumber +facsimiletelephonenumber: +1 313 555 9998 +facsimiletelephonenumber: +1 313 555 9999 + +dn: ou=Retired,ou=People,dc=example,dc=com +changetype: add +objectclass: organizationalUnit +ou: Retired + +dn: cn=Rosco P. Coltrane, ou=Information Technology Division, ou=People, dc=example,dc=com +changetype: add +objectclass: inetOrgPerson +cn: Rosco P. Coltrane +sn: Coltrane +uid: rosco +description: Fat tycoon + +dn: cn=Rosco P. Coltrane, ou=Information Technology Division, ou=People, dc=example,dc=com +changetype: modrdn +newrdn: cn=Rosco P. Coltrane +deleteoldrdn: 1 +newsuperior: ou=Retired,ou=People,dc=example,dc=com + +dn: cn=James A Jones 2, ou=Information Technology Division, ou=People, dc=example,dc=com +changetype: delete + +dn: ou=testdomain1,dc=example,dc=com +changetype: modrdn +newrdn: ou=itsdomain1 +deleteoldrdn: 1 + +dn: ou=itsdomain1,dc=example,dc=com +changetype: modify +replace: description +description: Example, Inc. ITS test domain + +dn: ou=testdomain2,dc=example,dc=com +changetype: modrdn +newrdn: ou=itsdomain2 +deleteoldrdn: 1 + +EOMODS + +RC=$? +if test $RC != 0 ; then + echo "ldapmodify failed ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +## ldappasswd test removed, not supported on DSEE + +echo "Stopping consumer to test recovery..." +kill -HUP $SLAVEPID +wait $SLAVEPID + +echo "Modifying more entries on the provider..." +$LDAPMODIFY -v -D "$DSEEDN" -h $LOCALHOST -p $PORT1 -w $DSEEPW >> \ + $TESTOUT 2>&1 << EOMODS +dn: cn=Rosco P. Coltrane, ou=Retired, ou=People, dc=example,dc=com +changetype: delete + +dn: cn=Bjorn Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com +changetype: modify +add: carLicense +carLicense: Mad Dog 20/20 + +dn: cn=Rosco P. Coltrane,ou=Retired,ou=People,dc=example,dc=com +changetype: add +objectclass: inetOrgPerson +sn: Coltrane +uid: rosco +cn: Rosco P. Coltrane + +dn: ou=itsdomain2,dc=example,dc=com +changetype: modify +replace: description +description: Example, Inc. itsdomain2 test domain + +# rename with a newly added newSuperior while the consumer is down (ITS#6472) +dn: ou=New Branch,dc=example,dc=com +changetype: add +objectClass: organizationalUnit +ou: New Branch + +dn: cn=Dorothy Stevens, ou=Alumni Association, ou=People, dc=example,dc=com +changetype: modrdn +newrdn: cn=Dorothy Stevens +deleteoldrdn: 0 +newsuperior: ou=New Branch,dc=example,dc=com + +EOMODS + +RC=$? +if test $RC != 0 ; then + echo "ldapmodify failed ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Restarting consumer..." +echo "RESTART" >> $LOG4 +$SLAPD -f $CONF4 -h $URI4 -d $LVL $TIMING >> $LOG4 2>&1 & +SLAVEPID=$! +if test $WAIT != 0 ; then + echo SLAVEPID $SLAVEPID + read foo +fi +KILLPIDS="$PID $SLAVEPID" + +echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..." +sleep $SLEEP1 + +echo "Using ldapsearch to read all the entries from the provider..." +$LDAPSEARCH -S "" -b "$BASEDN" -h $LOCALHOST -p $PORT1 \ + -D "$DSEEDN" -w "$DSEEPW" \ + '(objectclass=*)' '*' $OPATTRS > $MASTEROUT 2>&1 +RC=$? + +if test $RC != 0 ; then + echo "ldapsearch failed at provider ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Using ldapsearch to read all the entries from the consumer..." +$LDAPSEARCH -S "" -b "$BASEDN" -h $LOCALHOST -p $PORT4 \ + '(objectclass=*)' '*' $OPATTRS > $SLAVEOUT 2>&1 +RC=$? + +if test $RC != 0 ; then + echo "ldapsearch failed at consumer ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Filtering provider results..." +$LDIFFILTER -s a < $MASTEROUT > $MASTERFLT +echo "Filtering consumer results..." +$LDIFFILTER -s a < $SLAVEOUT > $SLAVEFLT + +echo "Comparing retrieved entries from provider and consumer..." +$CMP $MASTERFLT $SLAVEFLT > $CMPOUT + +if test $? != 0 ; then + echo "test failed - provider and consumer databases differ" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit 1 +fi + +test $KILLSERVERS != no && kill -HUP $KILLPIDS + +echo ">>>>> Test succeeded" + +test $KILLSERVERS != no && wait + +exit 0