]> git.ipfire.org Git - thirdparty/openldap.git/commitdiff
ITS#8966 add changelog support to syncrepl consumer
authorHoward Chu <hyc@openldap.org>
Mon, 28 Jan 2019 21:16:49 +0000 (21:16 +0000)
committerHoward Chu <hyc@openldap.org>
Wed, 30 Jan 2019 02:51:43 +0000 (18:51 -0800)
Tested against DSEE7. The DSEE binaries must be in your path to run the test script.

include/ldap.h
libraries/libldap/ldifutil.c
servers/slapd/schema/dsee.schema [new file with mode: 0644]
servers/slapd/syncrepl.c
tests/data/slapd-dsee-slave1.conf [new file with mode: 0644]
tests/data/test-dirsync-nocp.ldif
tests/scripts/defines.sh
tests/scripts/test072-dsee-sync [new file with mode: 0755]

index 91c614c89ed0ec844c1b82be06a81b22dad61787..0359a757ff757122c18f481405965e7759c14ff7 100644 (file)
@@ -2685,6 +2685,8 @@ ldap_parse_deref_control LDAP_P((
 #define LDIF_DEFAULT_ADD  0x01 /* if changetype missing, assume LDAP_ADD */
 #define LDIF_ENTRIES_ONLY 0x02 /* ignore changetypes other than add */
 #define LDIF_NO_CONTROLS  0x04 /* ignore control specifications */
+#define LDIF_MODS_ONLY    0x08 /* no changetypes, assume LDAP_MODIFY */
+#define LDIF_NO_DN        0x10 /* dn is not present */
 
 typedef struct ldifrecord {
        ber_tag_t lr_op; /* type of operation - LDAP_REQ_MODIFY, LDAP_REQ_ADD, etc. */
index c79bc8ce2f023219db4981bb98a2b652f99a31a3..d671e3cc3459c5525563d64cab6fe5a1f93686a1 100644 (file)
@@ -109,7 +109,7 @@ ldap_parse_ldif_record_x(
        char    *line, *dn;
        int             rc, modop;
        int             expect_modop, expect_sep;
-       int             ldapadd, new_entry, delete_entry, got_all;
+       int             ldapadd, new_entry, delete_entry, got_all, no_dn;
        LDAPMod **pmods;
        int version;
        LDAPControl **pctrls;
@@ -121,9 +121,11 @@ ldap_parse_ldif_record_x(
        memset( lr, 0, sizeof(LDIFRecord) );
        lr->lr_ctx = ctx; /* save memory context for later */
        ldapadd = flags & LDIF_DEFAULT_ADD;
+       no_dn = flags & LDIF_NO_DN;
+       expect_modop = flags & LDIF_MODS_ONLY;
        new_entry = ldapadd;
 
-       rc = got_all = delete_entry = modop = expect_modop = 0;
+       rc = got_all = delete_entry = modop = 0;
        expect_sep = 0;
        version = 0;
        pmods = NULL;
@@ -162,7 +164,7 @@ ldap_parse_ldif_record_x(
                }
                lr->lr_freeval[i] = freev;
 
-               if ( dn == NULL ) {
+               if ( dn == NULL && !no_dn ) {
                        if ( linenum+i == 1 && BV_CASEMATCH( lr->lr_btype+i, &BV_VERSION )) {
                                /* lutil_atoi() introduces a dependence of libldap
                                 * on liblutil; we only allow version 1 by now (ITS#6654)
@@ -190,7 +192,7 @@ ldap_parse_ldif_record_x(
        }
 
        /* check to make sure there was a dn: line */
-       if ( !dn ) {
+       if ( !dn && !no_dn ) {
                rc = 0;
                goto leave;
        }
@@ -207,27 +209,31 @@ ldap_parse_ldif_record_x(
                goto leave;
        }
 
-       i = idn+1;
-       /* Check for "control" tag after dn and before changetype. */
-       if ( BV_CASEMATCH( lr->lr_btype+i, &BV_CONTROL )) {
-               /* Parse and add it to the list of controls */
-               if ( !( flags & LDIF_NO_CONTROLS ) ) {
-                       rc = parse_ldif_control( lr->lr_vals+i, &pctrls );
-                       if (rc != 0) {
-                               fprintf( stderr,
-                                                _("%s: Error processing %s line, line %lu: %s\n"),
-                                                errstr, BV_CONTROL.bv_val, linenum+i, ldap_err2string(rc) );
+       if ( no_dn ) {
+               i = 0;
+       } else {
+               i = idn+1;
+               /* Check for "control" tag after dn and before changetype. */
+               if ( BV_CASEMATCH( lr->lr_btype+i, &BV_CONTROL )) {
+                       /* Parse and add it to the list of controls */
+                       if ( !( flags & LDIF_NO_CONTROLS ) ) {
+                               rc = parse_ldif_control( lr->lr_vals+i, &pctrls );
+                               if (rc != 0) {
+                                       fprintf( stderr,
+                                                        _("%s: Error processing %s line, line %lu: %s\n"),
+                                                        errstr, BV_CONTROL.bv_val, linenum+i, ldap_err2string(rc) );
+                               }
                        }
-               }
-               i++;
-               if ( i>= lr->lr_lines ) {
+                       i++;
+                       if ( i>= lr->lr_lines ) {
 short_input:
-                       fprintf( stderr,
-                               _("%s: Expecting more input after %s line, line %lu\n"),
-                               errstr, lr->lr_btype[i-1].bv_val, linenum+i );
-                       
-                       rc = LDAP_PARAM_ERROR;
-                       goto leave;
+                               fprintf( stderr,
+                                       _("%s: Expecting more input after %s line, line %lu\n"),
+                                       errstr, lr->lr_btype[i-1].bv_val, linenum+i );
+
+                               rc = LDAP_PARAM_ERROR;
+                               goto leave;
+                       }
                }
        }
 
@@ -421,7 +427,8 @@ short_input:
 
        lr->lr_mops = ber_memalloc_x( lr->lr_lines+1, ctx );
        lr->lr_mops[lr->lr_lines] = M_SEP;
-       lr->lr_mops[i-1] = M_SEP;
+       if ( i > 0 )
+               lr->lr_mops[i-1] = M_SEP;
 
        for ( ; i<lr->lr_lines; i++ ) {
                if ( expect_modop ) {
@@ -510,7 +517,8 @@ short_input:
        j = 0;
        k = -1;
        BER_BVZERO(&bv);
-       lr->lr_mops[idn-1] = M_SEP;
+       if ( idn > 0 )
+               lr->lr_mops[idn-1] = M_SEP;
        for (i=idn; i<lr->lr_lines; i++) {
                if ( lr->lr_mops[i] == M_SEP )
                        continue;
diff --git a/servers/slapd/schema/dsee.schema b/servers/slapd/schema/dsee.schema
new file mode 100644 (file)
index 0000000..b3ca144
--- /dev/null
@@ -0,0 +1,109 @@
+# $OpenLDAP$
+## This work is part of OpenLDAP Software <http://www.openldap.org/>.
+##
+## Copyright 2019 The OpenLDAP Foundation.
+## All rights reserved.
+##
+## Redistribution and use in source and binary forms, with or without
+## modification, are permitted only as authorized by the OpenLDAP
+## Public License.
+##
+## A copy of this license is available in the file LICENSE in the
+## top-level directory of the distribution or, alternatively, at
+## <http://www.OpenLDAP.org/license.html>.
+
+# This file is provided for informational purposes only.
+
+# These definitions are from Sun DSEE 7's cn=schema subentry.
+# None of the attributes had matching rules defined; we've
+# inserted usable ones as needed.
+
+# Some of these attributes are defined with NO-USER-MODIFICATION,
+# but slapd won't load such definitions from user-modifiable schema
+# files. So that designation has been removed, and commented accordingly.
+
+objectidentifier NetscapeRoot 2.16.840.1.113730
+objectidentifier NetscapeDS NetscapeRoot:3
+objectidentifier NSDSat        NetscapeDS:1
+objectidentifier NSDSoc NetscapeDS:2
+objectidentifier SunRoot 1.3.6.1.4.1.42
+objectidentifier SunDS SunRoot:2.27
+
+attributetype ( NSDSat:5
+       NAME 'changeNumber'
+       DESC 'Changelog attribute type'
+       EQUALITY integerMatch
+       SYNTAX 1.3.6.1.4.1.1466.115.121.1.27
+       X-ORIGIN 'Changelog Internet Draft' )
+
+attributetype ( NSDSat:6
+       NAME 'targetDn'
+       DESC 'Changelog attribute type'
+       EQUALITY distinguishedNameMatch
+       SYNTAX 1.3.6.1.4.1.1466.115.121.1.12
+       X-ORIGIN 'Changelog Internet Draft' )
+
+attributetype ( NSDSat:7
+       NAME 'changeType'
+       DESC 'Changelog attribute type'
+       SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
+       X-ORIGIN 'Changelog Internet Draft' )
+
+# They claim Binary syntax but it's really octetString
+attributetype ( NSDSat:8
+       NAME 'changes'
+       DESC 'Changelog attribute type'
+       SYNTAX 1.3.6.1.4.1.1466.115.121.1.5
+       X-ORIGIN 'Changelog Internet Draft' )
+
+attributetype ( NSDSat:9
+       NAME 'newRdn'
+       DESC 'Changelog attribute type'
+       EQUALITY distinguishedNameMatch
+       SYNTAX 1.3.6.1.4.1.1466.115.121.1.12
+       X-ORIGIN 'Changelog Internet Draft' )
+
+attributetype ( NSDSat:10
+       NAME 'deleteOldRdn'
+       DESC 'Changelog attribute type'
+       SYNTAX 1.3.6.1.4.1.1466.115.121.1.7
+       X-ORIGIN 'Changelog Internet Draft' )
+
+attributetype ( NSDSat:11
+       NAME 'newSuperior'
+       DESC 'Changelog attribute type'
+       EQUALITY distinguishedNameMatch
+       SYNTAX 1.3.6.1.4.1.1466.115.121.1.12
+       X-ORIGIN 'Changelog Internet Draft' )
+
+# should be generalizedTime, but they used directoryString instead...
+attributeType ( NSDSat:77
+       NAME 'changeTime'
+       DESC 'Sun ONE defined attribute type'
+       SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
+       X-ORIGIN 'Sun ONE Directory Server' )
+
+# These are UUIDs, but (of course) hyphenated differently than ours.
+# NO-USER-MODIFICATION
+attributetype ( NSDSat:542
+       NAME 'nsUniqueId'
+       DESC 'Sun ONE defined attribute type'
+       SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
+       SINGLE-VALUE
+       X-ORIGIN 'Sun ONE Directory Server' )
+
+# NO-USER-MODIFICATION
+attributeype ( SunDS:9.1.596
+       NAME 'targetUniqueId'
+       DESC 'RetroChangelog attribute type'
+       SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
+       SINGLE-VALUE
+       X-ORIGIN 'Sun Directory Server' )
+
+objectclass ( NSDSoc:1
+       NAME 'changeLogEntry'
+       DESC 'LDAP changelog objectclass'
+       SUP top STRUCTURAL
+       MUST ( targetDn $ changeTime $ changeNumber $ changeType )
+       MAY ( changes $ newRdn $ deleteOldRdn $ newSuperior )
+       X-ORIGIN 'Changelog Internet Draft' )
index 53bc6901c20aeadb6c13c0693f76b266a7134fcf..ee21b688b82cc1dde80221e506ad15eef65fd573 100644 (file)
@@ -50,6 +50,17 @@ static struct berval msad_addval = BER_BVC("range=1-1");
 static struct berval msad_delval = BER_BVC("range=0-0");
 #endif
 
+#ifdef LDAP_DEVEL
+#define DO_DSEE
+#endif
+
+#ifdef DO_DSEE
+static AttributeDescription *sy_ad_nsUniqueId;
+static AttributeDescription *sy_ad_dseeLastChange;
+
+#define DSEE_SYNC_ADD  0x20
+#endif
+
 #define        UUIDLEN 16
 
 struct nonpresent_entry {
@@ -144,6 +155,10 @@ typedef struct syncinfo_s {
 #endif
 #ifdef LDAP_CONTROL_X_DIRSYNC
        struct berval           si_dirSyncCookie;
+#endif
+#ifdef DO_DSEE
+       unsigned long   si_prevchange;;
+       unsigned long   si_lastchange;
 #endif
        ldap_pvt_thread_mutex_t si_mutex;
 } syncinfo_t;
@@ -179,6 +194,10 @@ static int syncrepl_dirsync_cookie(
                                        syncinfo_t *, Operation *, LDAPControl ** );
 #endif
 
+#ifdef DO_DSEE
+static int syncrepl_dsee_update( syncinfo_t *si, Operation *op ) ;
+#endif
+
 /* delta-mmr overlay handler */
 static int syncrepl_op_modify( Operation *op, SlapReply *rs );
 
@@ -188,6 +207,8 @@ static int nonpresent_callback( Operation *, SlapReply * );
 
 static AttributeDescription *sync_descs[4];
 
+static AttributeDescription *dsee_descs[7];
+
 /* delta-mmr */
 static AttributeDescription *ad_reqMod, *ad_reqDN;
 
@@ -199,6 +220,8 @@ typedef struct logschema {
        struct berval ls_delRdn;
        struct berval ls_newSup;
        struct berval ls_controls;
+       struct berval ls_uuid;
+       struct berval ls_changenum;
 } logschema;
 
 static logschema changelog_sc = {
@@ -208,7 +231,9 @@ static logschema changelog_sc = {
        BER_BVC("newRDN"),
        BER_BVC("deleteOldRDN"),
        BER_BVC("newSuperior"),
-       BER_BVC("controls")
+       BER_BVNULL,
+       BER_BVC("targetUniqueId"),
+       BER_BVC("changeNumber")
 };
 
 static logschema accesslog_sc = {
@@ -239,6 +264,10 @@ syncrepl_state2str( int state )
 #ifdef LDAP_CONTROL_X_DIRSYNC
        case MSAD_DIRSYNC_MODIFY:
                return "DIRSYNC_MOD";
+#endif
+#ifdef DO_DSEE
+       case DSEE_SYNC_ADD:
+               return "DSEE_ADD";
 #endif
        }
 
@@ -281,6 +310,20 @@ init_syncrepl(syncinfo_t *si)
                sync_descs[3] = NULL;
        }
 
+       if ( si->si_syncdata == SYNCDATA_CHANGELOG ) {
+               /* DSEE doesn't support allopattrs */
+               si->si_allopattrs = 0;
+               if ( !dsee_descs[0] ) {
+                       dsee_descs[0] = slap_schema.si_ad_objectClass;
+                       dsee_descs[1] = slap_schema.si_ad_creatorsName;
+                       dsee_descs[2] = slap_schema.si_ad_createTimestamp;
+                       dsee_descs[3] = slap_schema.si_ad_modifiersName;
+                       dsee_descs[4] = slap_schema.si_ad_modifyTimestamp;
+                       dsee_descs[5] = sy_ad_nsUniqueId;
+                       dsee_descs[6] = NULL;
+               }
+       }
+
        if ( si->si_allattrs && si->si_allopattrs )
                attrs = NULL;
        else
@@ -349,8 +392,10 @@ init_syncrepl(syncinfo_t *si)
                if ( si->si_allopattrs ) {
                        attrs[n++] = ch_strdup( sync_descs[0]->ad_cname.bv_val );
                } else {
-                       for ( i = 0; sync_descs[ i ] != NULL; i++ ) {
-                               attrs[ n++ ] = ch_strdup ( sync_descs[i]->ad_cname.bv_val );
+                       if ( si->si_syncdata != SYNCDATA_CHANGELOG ) {
+                               for ( i = 0; sync_descs[ i ] != NULL; i++ ) {
+                                       attrs[ n++ ] = ch_strdup ( sync_descs[i]->ad_cname.bv_val );
+                               }
                        }
                }
                attrs[ n ] = NULL;
@@ -376,6 +421,14 @@ init_syncrepl(syncinfo_t *si)
                }
                attrs[i] = NULL;
        }
+
+       if ( si->si_syncdata == SYNCDATA_CHANGELOG ) {
+               for ( n = 0; attrs[ n ] != NULL; n++ ) /* empty */;
+               attrs = ( char ** ) ch_realloc( attrs, (n + 6)*sizeof( char * ) );
+               for ( i = 0; dsee_descs[ i ] != NULL; i++ ) {
+                       attrs[ n++ ] = ch_strdup ( dsee_descs[i]->ad_cname.bv_val );
+               }
+       }
        
        si->si_attrs = attrs;
 
@@ -427,6 +480,8 @@ init_syncrepl(syncinfo_t *si)
        si->si_exattrs = exattrs;       
 }
 
+static struct berval generic_filterstr = BER_BVC("(objectclass=*)");
+
 static int
 ldap_sync_search(
        syncinfo_t *si,
@@ -442,6 +497,7 @@ ldap_sync_search(
        char *filter;
        int attrsonly;
        int scope;
+       char filterbuf[sizeof("(changeNumber>=18446744073709551615)")];
 
        /* setup LDAP SYNC control */
        ber_init2( ber, NULL, LBER_USE_DER );
@@ -450,8 +506,55 @@ ldap_sync_search(
        /* If we're using a log but we have no state, then fallback to
         * normal mode for a full refresh.
         */
-       if ( si->si_syncdata && !si->si_syncCookie.numcsns ) {
-               si->si_logstate = SYNCLOG_FALLBACK;
+       if ( si->si_syncdata ) {
+#ifdef DO_DSEE
+               if ( si->si_syncdata == SYNCDATA_CHANGELOG ) {
+                       LDAPMessage *res, *msg;
+                       unsigned long first = 0, last = 0;
+                       int gotfirst = 0, gotlast = 0;
+                       /* See if we're new enough for the remote server */
+                       lattrs[0] = "firstchangenumber";
+                       lattrs[1] = "lastchangenumber";
+                       lattrs[2] = NULL;
+                       rc = ldap_search_ext_s( si->si_ld, "", LDAP_SCOPE_BASE, generic_filterstr.bv_val, lattrs, 0,
+                               NULL, NULL, NULL, si->si_slimit, &res );
+                       if ( rc )
+                               return rc;
+                       msg = ldap_first_message( si->si_ld, res );
+                       if ( msg && ldap_msgtype( msg ) == LDAP_RES_SEARCH_ENTRY ) {
+                               BerElement *ber = NULL;
+                               struct berval bv, *bvals, **bvp = &bvals;;
+                               rc = ldap_get_dn_ber( si->si_ld, msg, &ber, &bv );
+                               for ( rc = ldap_get_attribute_ber( si->si_ld, msg, ber, &bv, bvp );
+                                       rc == LDAP_SUCCESS;
+                                       rc = ldap_get_attribute_ber( si->si_ld, msg, ber, &bv, bvp ) ) {
+                                       if ( bv.bv_val == NULL )
+                                               break;
+                                       if ( !strcasecmp( bv.bv_val, "firstchangenumber" )) {
+                                               first = strtoul( bvals[0].bv_val, NULL, 0 );
+                                               gotfirst = 1;
+                                       } else if ( !strcasecmp( bv.bv_val, "lastchangenumber" )) {
+                                               last = strtoul( bvals[0].bv_val, NULL, 0 );
+                                               gotlast = 1;
+                                       }
+                               }
+                       }
+                       ldap_msgfree( res );
+                       if ( gotfirst && gotlast ) {
+                               if ( !si->si_lastchange || si->si_lastchange < first )
+                                       si->si_logstate = SYNCLOG_FALLBACK;
+                               /* if we're in logging mode, it will update si_lastchange itself */
+                               if ( si->si_logstate == SYNCLOG_FALLBACK )
+                                       si->si_lastchange = last;
+                       } else {
+                               /* should be an error; changelog plugin not enabled on provider */
+                               si->si_logstate = SYNCLOG_FALLBACK;
+                       }
+               } else
+#endif
+               if ( si->si_logstate == SYNCLOG_LOGGING && !si->si_syncCookie.numcsns ) {
+                       si->si_logstate = SYNCLOG_FALLBACK;
+               }
        }
 
        /* Use the log parameters if we're in log mode */
@@ -467,16 +570,25 @@ ldap_sync_search(
                lattrs[3] = ls->ls_newRdn.bv_val;
                lattrs[4] = ls->ls_delRdn.bv_val;
                lattrs[5] = ls->ls_newSup.bv_val;
-               lattrs[6] = ls->ls_controls.bv_val;
-               lattrs[7] = slap_schema.si_ad_entryCSN->ad_cname.bv_val;
-               lattrs[8] = NULL;
+               if ( si->si_syncdata == SYNCDATA_ACCESSLOG ) {
+                       lattrs[6] = ls->ls_controls.bv_val;
+                       lattrs[7] = slap_schema.si_ad_entryCSN->ad_cname.bv_val;
+                       lattrs[8] = NULL;
+                       filter = si->si_logfilterstr.bv_val;
+                       scope = LDAP_SCOPE_SUBTREE;
+               } else {
+                       lattrs[6] = ls->ls_uuid.bv_val;
+                       lattrs[7] = ls->ls_changenum.bv_val;
+                       lattrs[8] = NULL;
+                       sprintf( filterbuf, "(changeNumber>=%lu)", si->si_lastchange+1 );
+                       filter = filterbuf;
+                       scope = LDAP_SCOPE_ONELEVEL;
+               }
 
                rhint = 0;
                base = si->si_logbase.bv_val;
-               filter = si->si_logfilterstr.bv_val;
                attrs = lattrs;
                attrsonly = 0;
-               scope = LDAP_SCOPE_SUBTREE;
        } else {
                rhint = 1;
                base = si->si_base.bv_val;
@@ -513,6 +625,11 @@ ldap_sync_search(
                        ctrls[1] = NULL;
                }
        } else
+#endif
+#ifdef DO_DSEE
+       if ( si->si_syncdata == SYNCDATA_CHANGELOG ) {
+               ctrls[0] = NULL;
+       } else
 #endif
        {
                if ( !BER_BVISNULL( &si->si_syncCookie.octet_str ) )
@@ -734,6 +851,23 @@ do_syncrep1(
                                si->si_dirSyncCookie = cookies[0];
                }
        } else
+#endif
+#ifdef DO_DSEE
+       if ( si->si_syncdata == SYNCDATA_CHANGELOG ) {
+               if ( !si->si_lastchange ) {
+                       BerVarray vals = NULL;
+
+                       op->o_req_ndn = si->si_contextdn;
+                       op->o_req_dn = op->o_req_ndn;
+                       /* try to read last change number */
+                       backend_attribute( op, NULL, &op->o_req_ndn,
+                               sy_ad_dseeLastChange, &vals, ACL_READ );
+                       if ( vals ) {
+                               si->si_lastchange = strtoul( vals[0].bv_val, NULL, 0 );
+                               si->si_prevchange = si->si_lastchange;
+                       }
+               }
+       } else
 #endif
        {
 
@@ -955,6 +1089,27 @@ do_syncrep2(
                                        goto done;
                                break;
                        }
+#endif
+#ifdef DO_DSEE
+                       if ( si->si_syncdata == SYNCDATA_CHANGELOG ) {
+                               if ( si->si_logstate == SYNCLOG_LOGGING ) {
+                                       rc = syncrepl_message_to_op( si, op, msg );
+                                       if ( rc )
+                                               si->si_logstate = SYNCLOG_FALLBACK;
+                               } else {
+                                       syncstate = DSEE_SYNC_ADD;
+                                       rc = syncrepl_message_to_entry( si, op, msg,
+                                               &modlist, &entry, syncstate, syncUUID );
+                                       if ( rc == 0 )
+                                               rc = syncrepl_entry( si, op, entry, &modlist, syncstate, syncUUID, NULL );
+                                       op->o_tmpfree( syncUUID[0].bv_val, op->o_tmpmemctx );
+                                       if ( modlist )
+                                               slap_mods_free( modlist, 1);
+                               }
+                               if ( rc )
+                                       goto done;
+                               break;
+                       }
 #endif
                        ldap_get_entry_controls( si->si_ld, msg, &rctrls );
                        ldap_get_dn_ber( si->si_ld, msg, NULL, &bdn );
@@ -1207,6 +1362,14 @@ do_syncrep2(
                                        "do_syncrep2: %s LDAP_RES_SEARCH_RESULT (%d) %s\n",
                                        si->si_ridtxt, err, ldap_err2string( err ) );
                        }
+                       if ( si->si_syncdata == SYNCDATA_CHANGELOG && err == LDAP_SUCCESS ) {
+                               rc = syncrepl_dsee_update( si, op );
+                               if ( rc == LDAP_SUCCESS ) {
+                                       rc = -2;        /* schedule a re-poll */
+                                       si->si_logstate = SYNCLOG_LOGGING;
+                               }
+                               goto done;
+                       }
                        if ( rctrls ) {
                                LDAPControl **next = NULL;
 #ifdef LDAP_CONTROL_X_DIRSYNC
@@ -1944,14 +2107,110 @@ syncrepl_accesslog_mods(
        return rc;
 }
 
+static int
+syncrepl_dsee_uuid(
+       struct berval *dseestr,
+       struct berval *syncUUID,
+       void *ctx
+)
+{
+       slap_mr_normalize_func *normf;
+       /* DSEE UUID is of form 12345678-12345678-12345678-12345678 */
+       if ( dseestr->bv_len != 35 )
+               return -1;
+       dseestr->bv_len++;
+       dseestr->bv_val[35] = '-';
+       normf = slap_schema.si_ad_entryUUID->ad_type->sat_equality->smr_normalize;
+       if ( normf( SLAP_MR_VALUE_OF_ATTRIBUTE_SYNTAX, NULL, NULL,
+               dseestr, &syncUUID[0], ctx ))
+               return -1;
+       (void)slap_uuidstr_from_normalized( &syncUUID[1], &syncUUID[0], ctx );
+       return LDAP_SUCCESS;
+}
+
 static int
 syncrepl_changelog_mods(
        syncinfo_t *si,
+       ber_tag_t req,
        struct berval *vals,
-       struct Modifications **modres
+       struct Modifications **modres,
+       struct berval *uuid,
+       void *ctx
 )
 {
-       return -1;      /* FIXME */
+       LDIFRecord lr;
+       struct berval rbuf = vals[0];
+       int i, rc;
+       int lrflags = LDIF_NO_DN;
+       Modifications *mod = NULL, *modlist = NULL, **modtail = &modlist;
+
+       if ( req == LDAP_REQ_ADD )
+               lrflags |= LDIF_ENTRIES_ONLY|LDIF_DEFAULT_ADD;
+       else
+               lrflags |= LDIF_MODS_ONLY;
+
+       rc = ldap_parse_ldif_record_x( &rbuf, 0, &lr, "syncrepl", lrflags, ctx );
+       for (i = 0; lr.lrop_mods[i] != NULL; i++) {
+               AttributeDescription *ad = NULL;
+               const char *text;
+               int j;
+               if ( slap_str2ad( lr.lrop_mods[i]->mod_type, &ad, &text ) ) {
+                       /* Invalid */
+                       Debug( LDAP_DEBUG_ANY, "syncrepl_changelog_mods: %s "
+                               "Invalid attribute %s, %s\n",
+                               si->si_ridtxt, lr.lrop_mods[i]->mod_type, text );
+                       slap_mods_free( modlist, 1 );
+                       modlist = NULL;
+                       rc = -1;
+                       break;
+               }
+               mod = (Modifications *) ch_malloc( sizeof( Modifications ) );
+               mod->sml_flags = 0;
+               mod->sml_op = lr.lrop_mods[i]->mod_op ^ LDAP_MOD_BVALUES;
+               mod->sml_next = NULL;
+               mod->sml_desc = ad;
+               mod->sml_type = ad->ad_cname;
+               mod->sml_values = NULL;
+               mod->sml_nvalues = NULL;
+               j = 0;
+               if ( lr.lrop_mods[i]->mod_bvalues != NULL ) {
+                       for (; lr.lrop_mods[i]->mod_bvalues[j] != NULL; j++ ) {
+                               struct berval bv, bv2;
+                               bv = *(lr.lrop_mods[i]->mod_bvalues[j]);
+                               REWRITE_VAL( si, ad, bv, bv2 );
+                               ber_bvarray_add( &mod->sml_values, &bv2 );
+                       }
+               }
+               mod->sml_numvals = j;
+
+               *modtail = mod;
+               modtail = &mod->sml_next;
+       }
+       ldap_ldif_record_done( &lr );
+
+       if ( req == LDAP_REQ_ADD && !BER_BVISNULL( uuid )) {
+               struct berval uuids[2];
+               if ( !syncrepl_dsee_uuid( uuid, uuids, ctx )) {
+                       mod = (Modifications *) ch_malloc( sizeof( Modifications ) );
+                       mod->sml_flags = 0;
+                       mod->sml_op = LDAP_MOD_ADD;
+                       mod->sml_next = NULL;
+                       mod->sml_desc = slap_schema.si_ad_entryUUID;
+                       mod->sml_type = slap_schema.si_ad_entryUUID->ad_cname;
+                       mod->sml_values = ch_malloc( 2 * sizeof(struct berval));
+                       mod->sml_nvalues = NULL;
+                       ber_dupbv( &mod->sml_values[0], &uuids[1] );
+                       BER_BVZERO( &mod->sml_values[1] );
+                       slap_sl_free( uuids[0].bv_val, ctx );
+                       slap_sl_free( uuids[1].bv_val, ctx );
+                       mod->sml_numvals = 1;
+                       *modtail = mod;
+                       modtail = &mod->sml_next;
+               }
+       }
+
+       *modres = modlist;
+       return rc;
 }
 
 typedef struct OpExtraSync {
@@ -2411,8 +2670,10 @@ syncrepl_message_to_op(
        struct berval   rdn = BER_BVNULL, sup = BER_BVNULL,
                prdn = BER_BVNULL, nrdn = BER_BVNULL,
                psup = BER_BVNULL, nsup = BER_BVNULL;
+       struct berval   dsee_uuid = BER_BVNULL, dsee_mods = BER_BVNULL;
        int             rc, deleteOldRdn = 0, freeReqDn = 0;
        int             do_graduate = 0;
+       unsigned long changenum = 0;
 
        if ( ldap_msgtype( msg ) != LDAP_RES_SEARCH_ENTRY ) {
                Debug( LDAP_DEBUG_ANY, "syncrepl_message_to_op: %s "
@@ -2483,7 +2744,7 @@ syncrepl_message_to_op(
                        if ( si->si_syncdata == SYNCDATA_ACCESSLOG ) {
                                rc = syncrepl_accesslog_mods( si, bvals, &modlist );
                        } else {
-                               rc = syncrepl_changelog_mods( si, bvals, &modlist );
+                               dsee_mods = bvals[0];
                        }
                        if ( rc ) goto done;
                } else if ( !ber_bvstrcasecmp( &bv, &ls->ls_newRdn ) ) {
@@ -2510,6 +2771,10 @@ syncrepl_message_to_op(
                                if ( !ber_bvcmp( &cbv, &rel_ctrl_bv ) )
                                        op->o_relax = SLAP_CONTROL_CRITICAL;
                        }
+               } else if ( !ber_bvstrcasecmp( &bv, &ls->ls_uuid ) ) {
+                       dsee_uuid = bvals[0];
+               } else if ( !ber_bvstrcasecmp( &bv, &ls->ls_changenum ) ) {
+                       changenum = strtoul( bvals->bv_val, NULL, 0 );
                } else if ( !ber_bvstrcasecmp( &bv,
                        &slap_schema.si_ad_entryCSN->ad_cname ) )
                {
@@ -2519,6 +2784,14 @@ syncrepl_message_to_op(
                ch_free( bvals );
        }
 
+       /* don't parse mods until we've gotten the uuid */
+       if ( si->si_syncdata == SYNCDATA_CHANGELOG && !BER_BVISNULL( &dsee_mods )) {
+               rc = syncrepl_changelog_mods( si, op->o_tag,
+                       &dsee_mods, &modlist, &dsee_uuid, op->o_tmpmemctx );
+               if ( rc )
+                       goto done;
+       }
+
        /* If we didn't get a mod type or a target DN, bail out */
        if ( op->o_tag == LBER_DEFAULT || BER_BVISNULL( &dn ) ) {
                rc = -1;
@@ -2640,6 +2913,9 @@ syncrepl_message_to_op(
                do_graduate = 0;
                break;
        }
+       if ( si->si_syncdata == SYNCDATA_CHANGELOG && !rc )
+               si->si_lastchange = changenum;
+
 done:
        if ( do_graduate )
                slap_graduate_commit_csn( op );
@@ -2721,13 +2997,15 @@ syncrepl_message_to_entry(
                return LDAP_OTHER;
        }
 
-       /* syncUUID[0] is normalized UUID received over the wire
-        * syncUUID[1] is denormalized UUID, generated here
-        */
-       (void)slap_uuidstr_from_normalized( &syncUUID[1], &syncUUID[0], op->o_tmpmemctx );
-       Debug( LDAP_DEBUG_SYNC,
-               "syncrepl_message_to_entry: %s DN: %s, UUID: %s\n",
-               si->si_ridtxt, bdn.bv_val, syncUUID[1].bv_val );
+       if ( si->si_syncdata != SYNCDATA_CHANGELOG ) {
+               /* syncUUID[0] is normalized UUID received over the wire
+                * syncUUID[1] is denormalized UUID, generated here
+                */
+               (void)slap_uuidstr_from_normalized( &syncUUID[1], &syncUUID[0], op->o_tmpmemctx );
+               Debug( LDAP_DEBUG_SYNC,
+                       "syncrepl_message_to_entry: %s DN: %s, UUID: %s\n",
+                       si->si_ridtxt, bdn.bv_val, syncUUID[1].bv_val );
+       }
 
        if ( syncstate == LDAP_SYNC_PRESENT || syncstate == LDAP_SYNC_DELETE ) {
                /* NOTE: this could be done even before decoding the DN,
@@ -2782,6 +3060,16 @@ syncrepl_message_to_entry(
                        continue;
                }
 
+               /* map nsUniqueId to entryUUID, drop nsUniqueId */
+               if ( si->si_syncdata == SYNCDATA_CHANGELOG &&
+                       !strcasecmp( tmp.sml_type.bv_val, sy_ad_nsUniqueId->ad_cname.bv_val )) {
+                       rc = syncrepl_dsee_uuid( &tmp.sml_values[0], syncUUID, op->o_tmpmemctx );
+                       ber_bvarray_free( tmp.sml_values );
+                       if ( rc )
+                               goto done;
+                       continue;
+               }
+
                mod  = (Modifications *) ch_malloc( sizeof( Modifications ) );
 
                mod->sml_op = LDAP_MOD_REPLACE;
@@ -2928,7 +3216,6 @@ syncrepl_dirsync_message(
                return LDAP_OTHER;
        }
 
-
        while ( ber_remaining( ber ) ) {
                AttributeDescription *ad = NULL;
 
@@ -3228,7 +3515,22 @@ static int syncrepl_dirsync_schema()
 }
 #endif /* LDAP_CONTROL_X_DIRSYNC */
 
-static struct berval generic_filterstr = BER_BVC("(objectclass=*)");
+#ifdef DO_DSEE
+static int syncrepl_dsee_schema()
+{
+       const char *text;
+       int rc;
+
+       rc = slap_str2ad( "nsUniqueId", &sy_ad_nsUniqueId, &text );
+       if ( rc )
+               return rc;
+       return register_at( "( 1.3.6.1.4.1.4203.666.1.28 "              /* OpenLDAP-specific */
+               "NAME 'lastChangeNumber' "
+               "DESC 'RetroChangelog latest change record' "
+               "SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 "
+               "SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation )", &sy_ad_dseeLastChange, 0);
+}
+#endif /* DO_DSEE */
 
 /* During a refresh, we may get an LDAP_SYNC_ADD for an already existing
  * entry if a previous refresh was interrupted before sending us a new
@@ -3511,6 +3813,7 @@ syncrepl_entry(
        switch ( syncstate ) {
        case LDAP_SYNC_ADD:
        case LDAP_SYNC_MODIFY:
+       case DSEE_SYNC_ADD:
                if ( BER_BVISNULL( &op->o_csn ))
                {
 
@@ -4352,6 +4655,63 @@ syncrepl_add_glue(
        return rc;
 }
 
+static int
+syncrepl_dsee_update(
+       syncinfo_t *si,
+       Operation *op
+)
+{
+       Backend *be = op->o_bd;
+       Modifications mod;
+       struct berval first = BER_BVNULL;
+       slap_callback cb = { NULL };
+       SlapReply       rs_modify = {REP_RESULT};
+       char valbuf[sizeof("18446744073709551615")];
+       struct berval bvals[2];
+       int rc;
+
+       if ( si->si_lastchange == si->si_prevchange )
+               return 0;
+
+       mod.sml_op = LDAP_MOD_REPLACE;
+       mod.sml_desc = sy_ad_dseeLastChange;
+       mod.sml_type = mod.sml_desc->ad_cname;
+       mod.sml_flags = SLAP_MOD_INTERNAL;
+       mod.sml_nvalues = NULL;
+       mod.sml_values = bvals;
+       mod.sml_numvals = 1;
+       mod.sml_next = NULL;
+       bvals[0].bv_val = valbuf;
+       bvals[0].bv_len = sprintf( valbuf, "%lu", si->si_lastchange );
+       BER_BVZERO( &bvals[1] );
+
+       op->o_bd = si->si_wbe;
+
+       op->o_tag = LDAP_REQ_MODIFY;
+
+       cb.sc_response = syncrepl_null_callback;
+       cb.sc_private = si;
+
+       op->o_callback = &cb;
+       op->o_req_dn = si->si_contextdn;
+       op->o_req_ndn = si->si_contextdn;
+
+       /* update contextCSN */
+       op->o_dont_replicate = 1;
+
+       /* avoid timestamp collisions */
+       slap_op_time( &op->o_time, &op->o_tincr );
+
+       op->orm_modlist = &mod;
+       op->orm_no_opattrs = 1;
+       rc = op->o_bd->be_modify( op, &rs_modify );
+
+       op->o_bd = be;
+       si->si_prevchange = si->si_lastchange;
+
+       return rc;
+}
+
 static int
 syncrepl_updateCookie(
        syncinfo_t *si,
@@ -5805,6 +6165,24 @@ parse_syncrepl_line(
                        val = c->argv[ i ] + STRLENOF( SYNCDATASTR "=" );
                        si->si_syncdata = verb_to_mask( val, datamodes );
                        si->si_got |= GOT_SYNCDATA;
+                       if ( si->si_syncdata == SYNCDATA_CHANGELOG ) {
+#ifdef DO_DSEE
+                               if ( sy_ad_nsUniqueId == NULL ) {
+                                       int rc = syncrepl_dsee_schema();
+                                       if ( rc ) {
+                                               snprintf( c->cr_msg, sizeof( c->cr_msg ),
+                                                       "changelog schema problem (%d)\n", rc );
+                                               Debug( LDAP_DEBUG_ANY, "%s: %s.\n", c->log, c->cr_msg, 0 );
+                                               return 1;
+                                       }
+                               }
+#else
+                               snprintf( c->cr_msg, sizeof( c->cr_msg ),
+                                       "changelog not yet supported\n" );
+                               Debug( LDAP_DEBUG_ANY, "%s: %s.\n", c->log, c->cr_msg, 0 );
+                               return 1;
+#endif
+                       }
                } else if ( !strncasecmp( c->argv[ i ], STRICT_REFRESH,
                                        STRLENOF( STRICT_REFRESH ) ) )
                {
diff --git a/tests/data/slapd-dsee-slave1.conf b/tests/data/slapd-dsee-slave1.conf
new file mode 100644 (file)
index 0000000..d26ec75
--- /dev/null
@@ -0,0 +1,63 @@
+# slave slapd config -- for testing of SYNC replication
+# $OpenLDAP$
+## This work is part of OpenLDAP Software <http://www.openldap.org/>.
+##
+## Copyright 1998-2018 The OpenLDAP Foundation.
+## All rights reserved.
+##
+## Redistribution and use in source and binary forms, with or without
+## modification, are permitted only as authorized by the OpenLDAP
+## Public License.
+##
+## A copy of this license is available in the file LICENSE in the
+## top-level directory of the distribution or, alternatively, at
+## <http://www.OpenLDAP.org/license.html>.
+
+include                @SCHEMADIR@/core.schema
+include                @SCHEMADIR@/cosine.schema
+include                @SCHEMADIR@/inetorgperson.schema
+include                @SCHEMADIR@/openldap.schema
+include                @SCHEMADIR@/nis.schema
+include                @SCHEMADIR@/dsee.schema
+#
+pidfile                @TESTDIR@/slapd.2.pid
+argsfile       @TESTDIR@/slapd.2.args
+
+#mod#modulepath        ../servers/slapd/back-@BACKEND@/
+#mod#moduleload        back_@BACKEND@.la
+#monitormod#modulepath ../servers/slapd/back-monitor/
+#monitormod#moduleload back_monitor.la
+
+#######################################################################
+# consumer database definitions
+#######################################################################
+
+database       @BACKEND@
+suffix         "dc=example,dc=com"
+rootdn         "cn=Replica,dc=example,dc=com"
+rootpw         secret
+#null#bind             on
+#~null~#directory      @TESTDIR@/db.2.a
+#indexdb#index         objectClass     eq
+#indexdb#index         cn,sn,uid       pres,eq,sub
+#indexdb#index         entryUUID,entryCSN      eq
+#ndb#dbname db_2
+#ndb#include @DATADIR@/ndb.conf
+
+# Don't change syncrepl spec yet
+syncrepl       rid=1
+               provider=@URI1@
+               binddn="cn=Directory Manager"
+               bindmethod=simple
+               credentials=secret21
+               searchbase="dc=example,dc=com"
+               filter="(objectClass=*)"
+               schemachecking=off
+               scope=sub
+               type=refreshOnly
+               logbase="cn=changelog"
+               syncdata=changelog
+               retry="3 +" interval=00:00:00:03
+updateref      @URI1@
+
+#monitor#database      monitor
index 55ea709130ebc4cb7d00fb7c1b73b4ca0a556e3d..daeda7979aadbd95d8408dc1f159d6e86cecb2e9 100644 (file)
@@ -27,7 +27,7 @@ description: MSAD doesn't like long descriptions
 description: 5K and 3K are too big
 
 dn: cn=Barbara Jensen,ou=Information Technology Division,ou=People,dc=example,dc=com
-objectclass: inetorgperson
+objectclass: inetOrgPerson
 objectclass: domainRelatedObject
 cn: Barbara Jensen
 sn:: IEplbnNlbiA=
@@ -47,7 +47,7 @@ telephonenumber: +1 313 555 9022
 associatedDomain: test.openldap.org
 
 dn: cn=Bjorn Jensen,ou=Information Technology Division,ou=People,dc=example,dc=com
-objectclass: inetorgperson
+objectclass: inetOrgPerson
 objectclass: domainRelatedObject
 cn: Bjorn Jensen
 sn: Jensen
@@ -66,7 +66,7 @@ telephonenumber: +1 313 555 0355
 associatedDomain: test.openldap.org
 
 dn: cn=Dorothy Stevens,ou=Alumni Association,ou=People,dc=example,dc=com
-objectclass: inetorgperson
+objectclass: inetOrgPerson
 objectclass: domainRelatedObject
 cn: Dorothy Stevens
 sn: Stevens
@@ -83,7 +83,7 @@ homephone: +1 313 555 0454
 associatedDomain: test.openldap.org
 
 dn: cn=James A Jones 1,ou=Alumni Association,ou=People,dc=example,dc=com
-objectclass: inetorgperson
+objectclass: inetOrgPerson
 objectclass: domainRelatedObject
 cn: James A Jones 1
 sn: Jones
@@ -101,7 +101,7 @@ telephonenumber: +1 313 555 0895
 associatedDomain: test.openldap.org
 
 dn: cn=James A Jones 2,ou=Information Technology Division,ou=People,dc=example,dc=com
-objectclass: inetorgperson
+objectclass: inetOrgPerson
 objectclass: domainRelatedObject
 cn: James A Jones 2
 sn: Doe
@@ -118,7 +118,7 @@ telephonenumber: +1 313 555 7334
 associatedDomain: test.openldap.org
 
 dn: cn=Jane Doe,ou=Alumni Association,ou=People,dc=example,dc=com
-objectclass: inetorgperson
+objectclass: inetOrgPerson
 objectclass: domainRelatedObject
 cn: Jane Doe
 sn: Doe
@@ -136,7 +136,7 @@ telephonenumber: +1 313 555 4774
 associatedDomain: test.openldap.org
 
 dn: cn=Jennifer Smith,ou=Alumni Association,ou=People,dc=example,dc=com
-objectclass: inetorgperson
+objectclass: inetOrgPerson
 objectclass: domainRelatedObject
 cn: Jennifer Smith
 sn: Smith
@@ -153,7 +153,7 @@ telephonenumber: +1 313 555 8232
 associatedDomain: test.openldap.org
 
 dn: cn=John Doe,ou=Information Technology Division,ou=People,dc=example,dc=com
-objectclass: inetorgperson
+objectclass: inetOrgPerson
 objectclass: domainRelatedObject
 cn: John Doe
 sn: Doe
@@ -170,7 +170,7 @@ telephonenumber: +1 313 555 9394
 associatedDomain: test.openldap.org
 
 dn: cn=Manager,dc=example,dc=com
-objectclass: inetorgperson
+objectclass: inetOrgPerson
 objectclass: domainRelatedObject
 cn: Manager
 sn: Manager
@@ -179,7 +179,7 @@ userpassword:: c2VjcmV0
 associatedDomain: test.openldap.org
 
 dn: cn=Mark Elliot,ou=Alumni Association,ou=People,dc=example,dc=com
-objectclass: inetorgperson
+objectclass: inetOrgPerson
 objectclass: domainRelatedObject
 cn: Mark Elliot
 sn: Elliot
@@ -196,7 +196,7 @@ telephonenumber: +1 313 555 4177
 associatedDomain: test.openldap.org
 
 dn: cn=Ursula Hampster,ou=Alumni Association,ou=People,dc=example,dc=com
-objectclass: inetorgperson
+objectclass: inetOrgPerson
 objectclass: domainRelatedObject
 cn: Ursula Hampster
 sn: Hampster
@@ -226,7 +226,7 @@ member: cn=Bjorn Jensen,ou=Information Technology Division,ou=People,dc=example,
 owner: cn=Manager,dc=example,dc=com
 cn: All Staff
 description: Everyone in the sample data
-objectclass: groupofnames
+objectclass: groupOfNames
 objectclass: domainRelatedObject
 associatedDomain: test.openldap.org
 
@@ -234,7 +234,7 @@ dn: cn=ITD Staff,ou=Groups,dc=example,dc=com
 owner: cn=Manager,dc=example,dc=com
 description: All ITD Staff
 cn: ITD Staff
-objectclass: groupofuniquenames
+objectclass: groupOfUniqueNames
 objectclass: domainRelatedObject
 uniquemember: cn=Manager,dc=example,dc=com
 uniquemember: cn=Bjorn Jensen,OU=Information Technology Division,ou=People,dc=example,dc=com 
@@ -253,7 +253,7 @@ member: cn=Ursula Hampster,ou=Alumni Association,ou=People,dc=example,dc=com
 owner: cn=Manager,dc=example,dc=com
 description: All Alumni Assoc Staff
 cn: Alumni Assoc Staff
-objectclass: groupofnames
+objectclass: groupOfNames
 objectclass: domainRelatedObject
 associatedDomain: test.openldap.org
 
index 565b72fd0cc2aee14220f06131957a3071a5b8a2..6a8b103dbf53bd6ac1075c33b4deccc6315f3169 100755 (executable)
@@ -106,6 +106,7 @@ P1SRSLAVECONF=$DATADIR/slapd-syncrepl-slave-persist1.conf
 P2SRSLAVECONF=$DATADIR/slapd-syncrepl-slave-persist2.conf
 P3SRSLAVECONF=$DATADIR/slapd-syncrepl-slave-persist3.conf
 DIRSYNC1CONF=$DATADIR/slapd-dirsync1.conf
+DSEESYNC1CONF=$DATADIR/slapd-dsee-slave1.conf
 REFSLAVECONF=$DATADIR/slapd-ref-slave.conf
 SCHEMACONF=$DATADIR/slapd-schema.conf
 TLSCONF=$DATADIR/slapd-tls.conf
diff --git a/tests/scripts/test072-dsee-sync b/tests/scripts/test072-dsee-sync
new file mode 100755 (executable)
index 0000000..367b084
--- /dev/null
@@ -0,0 +1,330 @@
+#! /bin/sh
+# $OpenLDAP$
+## This work is part of OpenLDAP Software <http://www.openldap.org/>.
+##
+## Copyright 1998-2018 The OpenLDAP Foundation.
+## All rights reserved.
+##
+## Redistribution and use in source and binary forms, with or without
+## modification, are permitted only as authorized by the OpenLDAP
+## Public License.
+##
+## A copy of this license is available in the file LICENSE in the
+## top-level directory of the distribution or, alternatively, at
+## <http://www.OpenLDAP.org/license.html>.
+
+echo "running defines.sh"
+. $SRCDIR/scripts/defines.sh
+
+if test -z `which dsadm`; then
+       echo "DSEE dsadm not in path, test skipped"
+       exit 0
+fi 
+
+mkdir -p $TESTDIR $DBDIR2
+
+#
+# Test replication:
+# - start provider
+# - start consumer
+# - populate over ldap
+# - perform some modifies and deleted
+# - attempt to modify the consumer (referral)
+# - retrieve database over ldap and compare against expected results
+#
+
+DSEEPW=secret21
+DSEEDN="cn=Directory Manager"
+DSEEPWF=$TESTDIR/dseepw
+
+echo "secret21" > $DSEEPWF
+
+echo "Setting up DSEE provider slapd on TCP/IP port $PORT1..."
+dsadm create -p $PORT1 -w $DSEEPWF $DBDIR1
+dsadm start $DBDIR1
+dsconf create-suffix -c -p $PORT1 -w $DSEEPWF $BASEDN
+dsconf set-server-prop -p $PORT1 -w $DSEEPWF moddn-enabled:on
+dsconf set-server-prop -p $PORT1 -w $DSEEPWF retro-cl-enabled:on
+dsadm restart $DBDIR1
+KILLPIDS=`basename $DBDIR1/locks/server/*`
+
+sleep 1
+
+echo "Using ldapsearch to check that provider slapd is running..."
+for i in 0 1 2 3 4 5; do
+       $LDAPSEARCH -s base -b "$BASEDN" -h $LOCALHOST -p $PORT1 \
+               'objectclass=*' > /dev/null 2>&1
+       RC=$?
+       if test $RC = 0 ; then
+               break
+       fi
+       echo "Waiting 5 seconds for slapd to start..."
+       sleep 5
+done
+
+if test $RC != 0 ; then
+       echo "ldapsearch failed ($RC)!"
+       test $KILLSERVERS != no && kill -HUP $KILLPIDS
+       exit $RC
+fi
+
+echo "Starting consumer slapd on TCP/IP port $PORT2..."
+. $CONFFILTER $BACKEND $MONITORDB < $DSEESYNC1CONF > $CONF2
+$SLAPD -f $CONF2 -h $URI2 -d $LVL $TIMING > $LOG2 2>&1 &
+SLAVEPID=$!
+if test $WAIT != 0 ; then
+    echo SLAVEPID $SLAVEPID
+    read foo
+fi
+KILLPIDS="$KILLPIDS $SLAVEPID"
+
+sleep 1
+
+echo "Using ldapsearch to check that consumer slapd is running..."
+for i in 0 1 2 3 4 5; do
+       $LDAPSEARCH -s base -b "$MONITOR" -h $LOCALHOST -p $PORT2 \
+               'objectclass=*' > /dev/null 2>&1
+       RC=$?
+       if test $RC = 0 ; then
+               break
+       fi
+       echo "Waiting 5 seconds for slapd to start..."
+       sleep 5
+done
+
+if test $RC != 0 ; then
+       echo "ldapsearch failed ($RC)!"
+       test $KILLSERVERS != no && kill -HUP $KILLPIDS
+       exit $RC
+fi
+
+# using LDIFDIRSYNCNOCP to avoid custom OpenLDAP schema
+echo "Using ldapadd to populate the provider directory..."
+$LDAPADD -D "$DSEEDN" -h $LOCALHOST -p $PORT1 -w $DSEEPW < \
+       $LDIFDIRSYNCNOCP > /dev/null 2>&1
+RC=$?
+if test $RC != 0 ; then
+       echo "ldapadd failed ($RC)!"
+       test $KILLSERVERS != no && kill -HUP $KILLPIDS
+       exit $RC
+fi
+
+echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
+sleep $SLEEP1
+
+echo "Using ldapmodify to modify provider directory..."
+
+#
+# Do some modifications
+#
+
+$LDAPMODIFY -v -D "$DSEEDN" -h $LOCALHOST -p $PORT1 -w $DSEEPW > \
+       $TESTOUT 2>&1 << EOMODS
+dn: cn=James A Jones 1, ou=Alumni Association, ou=People, dc=example,dc=com
+changetype: modify
+add: carLicense
+carLicense: Orange Juice
+-
+delete: sn
+sn: Jones
+-
+add: sn
+sn: Jones
+
+dn: cn=Bjorn Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com
+changetype: modify
+replace: carLicense
+carLicense: Iced Tea
+carLicense: Mad Dog 20/20
+
+dn: cn=ITD Staff,ou=Groups,dc=example,dc=com
+changetype: modify
+delete: uniquemember
+uniquemember: cn=James A Jones 2, ou=Information Technology Division, ou=People, dc=example,dc=com
+uniquemember: cn=Bjorn Jensen, ou=Information Technology Division, ou=People, dc=example,dc=com
+-
+add: uniquemember
+uniquemember: cn=Dorothy Stevens,ou=Alumni Association,ou=People,dc=example,dc=com
+uniquemember: cn=James A Jones 1,ou=Alumni Association,ou=People,dc=example,dc=com
+
+dn: cn=All Staff,ou=Groups,dc=example,dc=com
+changetype: modify
+delete: description
+
+dn: cn=Gern Jensen,ou=Information Technology Division,ou=People,dc=example,dc=com
+changetype: add
+objectclass: inetOrgPerson
+cn: Gern Jensen
+sn: Jensen
+uid: gjensen
+title: Chief Investigator, ITD
+postaladdress: ITD $ 535 W. William St $ Ann Arbor, MI 48103
+seealso: cn=All Staff,ou=Groups,dc=example,dc=com
+carLicense: Coffee
+homepostaladdress: 844 Brown St. Apt. 4 $ Ann Arbor, MI 48104
+description: Very odd
+facsimiletelephonenumber: +1 313 555 7557
+telephonenumber: +1 313 555 8343
+mail: gjensen@mailgw.example.com
+homephone: +1 313 555 8844
+
+dn: ou=Retired,ou=People,dc=example,dc=com
+changetype: add
+objectclass: organizationalUnit
+ou: Retired
+
+dn: cn=Rosco P. Coltrane, ou=Information Technology Division, ou=People, dc=example,dc=com
+changetype: add
+objectclass: inetOrgPerson
+cn: Rosco P. Coltrane
+sn: Coltrane
+uid: rosco
+
+dn: cn=Rosco P. Coltrane, ou=Information Technology Division, ou=People, dc=example,dc=com
+changetype: modrdn
+newrdn: cn=Rosco P. Coltrane
+deleteoldrdn: 1
+newsuperior: ou=Retired,ou=People,dc=example,dc=com
+
+dn: cn=James A Jones 2, ou=Information Technology Division, ou=People, dc=example,dc=com
+changetype: delete
+
+dn: ou=testdomain1,dc=example,dc=com
+changetype: modrdn
+newrdn: ou=itsdomain1
+deleteoldrdn: 1
+
+dn: ou=itsdomain1,dc=example,dc=com
+changetype: modify
+replace: description
+description: Example, Inc. ITS test domain
+
+EOMODS
+
+RC=$?
+if test $RC != 0 ; then
+       echo "ldapmodify failed ($RC)!"
+       test $KILLSERVERS != no && kill -HUP $KILLPIDS
+       exit $RC
+fi
+
+echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
+sleep $SLEEP1
+
+echo "Performing modrdn alone on the provider..."
+$LDAPMODIFY -v -D "$DSEEDN" -h $LOCALHOST -p $PORT1 -w $DSEEPW > \
+       $TESTOUT 2>&1 << EOMODS
+dn: ou=testdomain2,dc=example,dc=com
+changetype: modrdn
+newrdn: ou=itsdomain2
+deleteoldrdn: 1
+
+EOMODS
+
+RC=$?
+if test $RC != 0 ; then
+       echo "ldapmodify failed ($RC)!"
+       test $KILLSERVERS != no && kill -HUP $KILLPIDS
+       exit $RC
+fi
+
+echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
+sleep $SLEEP1
+
+echo "Performing modify alone on the provider..."
+$LDAPMODIFY -v -D "$DSEEDN" -h $LOCALHOST -p $PORT1 -w $DSEEPW > \
+       $TESTOUT 2>&1 << EOMODS
+dn: ou=itsdomain2,dc=example,dc=com
+changetype: modify
+replace: description
+description: Example, Inc. itsdomain2 test domain
+
+EOMODS
+
+RC=$?
+if test $RC != 0 ; then
+       echo "ldapmodify failed ($RC)!"
+       test $KILLSERVERS != no && kill -HUP $KILLPIDS
+       exit $RC
+fi
+
+echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
+sleep $SLEEP1
+
+echo "Performing larger modify on the provider..."
+$LDAPMODIFY -v -D "$DSEEDN" -h $LOCALHOST -p $PORT1 -w $DSEEPW > \
+       $TESTOUT 2>&1 << EOMODS
+dn: cn=Alumni Assoc Staff,ou=Groups,dc=example,dc=com
+changetype: modify
+replace: cn
+cn: Alumni Assoc Staff
+-
+replace: description
+description: blablabla
+-
+replace: member
+member: cn=Manager,dc=example,dc=com
+member: cn=Dorothy Stevens,ou=Alumni Association,ou=People,dc=example,dc=com
+member: cn=James A Jones 1,ou=Alumni Association,ou=People,dc=example,dc=com
+member: cn=Jane Doe,ou=Alumni Association,ou=People,dc=example,dc=com
+member: cn=Jennifer Smith,ou=Alumni Association,ou=People,dc=example,dc=com
+member: cn=Mark Elliot,ou=Alumni Association,ou=People,dc=example,dc=com
+member: cn=Ursula Hampster,ou=Alumni Association,ou=People,dc=example,dc=com
+
+EOMODS
+
+RC=$?
+if test $RC != 0 ; then
+       echo "ldapmodify failed ($RC)!"
+       test $KILLSERVERS != no && kill -HUP $KILLPIDS
+       exit $RC
+fi
+
+echo "Waiting $SLEEP1 seconds for syncrepl to receive changes..."
+sleep $SLEEP1
+
+OPATTRS="creatorsName createTimestamp modifiersName modifyTimestamp"
+
+echo "Using ldapsearch to read all the entries from the provider..."
+$LDAPSEARCH -S "" -b "$BASEDN" -h $LOCALHOST -p $PORT1 \
+       -D "$DSEEDN" -w $DSEEPW \
+       '(objectclass=*)' '*' $OPATTRS > $MASTEROUT 2>&1
+RC=$?
+
+if test $RC != 0 ; then
+       echo "ldapsearch failed at provider ($RC)!"
+       test $KILLSERVERS != no && kill -HUP $KILLPIDS
+       exit $RC
+fi
+
+echo "Using ldapsearch to read all the entries from the consumer..."
+$LDAPSEARCH -S "" -b "$BASEDN" -h $LOCALHOST -p $PORT2 \
+       '(objectclass=*)' '*' $OPATTRS > $SLAVEOUT 2>&1
+RC=$?
+
+if test $RC != 0 ; then
+       echo "ldapsearch failed at consumer ($RC)!"
+       test $KILLSERVERS != no && kill -HUP $KILLPIDS
+       exit $RC
+fi
+
+test $KILLSERVERS != no && kill -HUP $KILLPIDS
+
+echo "Filtering provider results..."
+$LDIFFILTER -s a < $MASTEROUT > $MASTERFLT
+echo "Filtering consumer results..."
+$LDIFFILTER -s a < $SLAVEOUT > $SLAVEFLT
+
+echo "Comparing retrieved entries from provider and consumer..."
+$CMP $MASTERFLT $SLAVEFLT > $CMPOUT
+
+if test $? != 0 ; then
+       echo "test failed - provider and consumer databases differ"
+       exit 1
+fi
+
+echo ">>>>> Test succeeded"
+
+test $KILLSERVERS != no && wait
+
+exit 0