]> git.ipfire.org Git - thirdparty/kea.git/commitdiff
[master] Merge branch 'trac3984' (v4 declined lease recovery)
authorTomek Mrugalski <tomasz@isc.org>
Thu, 1 Oct 2015 12:23:35 +0000 (14:23 +0200)
committerTomek Mrugalski <tomasz@isc.org>
Thu, 1 Oct 2015 12:23:35 +0000 (14:23 +0200)
Conflicts:
src/lib/dhcpsrv/alloc_engine.cc
src/lib/dhcpsrv/tests/alloc_engine_expiration_unittest.cc

1  2 
src/lib/dhcpsrv/alloc_engine.cc
src/lib/dhcpsrv/alloc_engine.h
src/lib/dhcpsrv/alloc_engine_messages.mes
src/lib/dhcpsrv/lease.cc
src/lib/dhcpsrv/tests/alloc_engine_expiration_unittest.cc
src/lib/dhcpsrv/tests/memfile_lease_mgr_unittest.cc

index ce00a2269381a7936e51de77bdfe4ba7e8352666,1f17447e1038d34fa6b8962e08eae800e5c2f9cf..08cc2ffd7f04d1f7f83e9ce2a6ae16892af48cfb
@@@ -1435,44 -1387,42 +1435,60 @@@ AllocEngine::reclaimExpiredLeases4(cons
      BOOST_FOREACH(Lease4Ptr lease, leases) {
  
          try {
 -            /// @todo execute a lease4_expire hook here.
 +            // The skip flag indicates if the callouts have taken responsibility
 +            // for reclaiming the lease. The callout will set this to true if
 +            // it reclaims the lease itself. In this case the reclamation routine
 +            // will not update DNS nor update the database.
 +            bool skipped = false;
 +            if (callout_handle) {
 +                callout_handle->deleteAllArguments();
 +                callout_handle->setArgument("lease4", lease);
 +                callout_handle->setArgument("remove_lease", remove_lease);
 +
 +                HooksManager::callCallouts(Hooks.hook_index_lease4_expire_,
 +                                           *callout_handle);
 +
 +                skipped = callout_handle->getSkip();
 +            }
  
 -            // Generate removal name change request for D2, if required.
 -            // This will return immediatelly if the DNS wasn't updated
 -            // when the lease was created.
 -            if (lease->client_id_) {
 -                // Client id takes precedence over HW address.
 -                queueRemovalNameChangeRequest(lease, lease->client_id_->getClientId());
 +            if (!skipped) {
 +                // Generate removal name change request for D2, if required.
 +                // This will return immediatelly if the DNS wasn't updated
 +                // when the lease was created.
 +                if (lease->client_id_) {
 +                    // Client id takes precedence over HW address.
 +                    queueRemovalNameChangeRequest(lease, lease->client_id_->getClientId());
  
 -            } else {
 -                // Client id is not specified for the lease. Use HW address
 -                // instead.
 -                queueRemovalNameChangeRequest(lease, lease->hwaddr_);
 -            }
 +                } else {
 +                    // Client id is not specified for the lease. Use HW address
 +                    // instead.
 +                    queueRemovalNameChangeRequest(lease, lease->hwaddr_);
 +                }
  
 -            // Let's check if the lease that just expired is in DECLINED state.
 -            // If it is, we need to conduct couple extra steps and also force
 -            // its removal.
 -            bool remove_tmp = remove_lease;
 -            if (lease->state_ == Lease::STATE_DECLINED) {
 -                // There's no point in keeping declined lease after its
 -                // reclaimation. Declined lease doesn't have any client
 -                // identifying information anymore.
 -                remove_tmp = true;
 -
 -                // Do extra steps required for declined lease reclaimation:
 -                // - bump decline-related stats
 -                // - log separate message
 -                reclaimDeclined(lease);
++                // Let's check if the lease that just expired is in DECLINED state.
++                // If it is, we need to conduct couple extra steps and also force
++                // its removal.
++                bool remove_tmp = remove_lease;
++                if (lease->state_ == Lease::STATE_DECLINED) {
++                    // There's no point in keeping declined lease after its
++                    // reclaimation. Declined lease doesn't have any client
++                    // identifying information anymore.
++                    remove_tmp = true;
++
++                    // Do extra steps required for declined lease reclaimation:
++                    // - bump decline-related stats
++                    // - log separate message
++                    reclaimDeclined(lease);
++                }
++
 +                // Reclaim the lease - depending on the configuration, set the
 +                // expired-reclaimed state or simply remove it.
-                 reclaimLeaseInDatabase<Lease4Ptr>(lease, remove_lease,
++                reclaimLeaseInDatabase<Lease4Ptr>(lease, remove_tmp,
 +                                                  boost::bind(&LeaseMgr::updateLease4,
 +                                                              &lease_mgr, _1));
              }
  
 -            // Reclaim the lease - depending on the configuration, set the
 -            // expired-reclaimed state or simply remove it.
 -            reclaimLeaseInDatabase<Lease4Ptr>(lease, remove_tmp,
 -                                              boost::bind(&LeaseMgr::updateLease4,
 -                                                          &lease_mgr, _1));
 +            ++leases_processed;
  
              // Update statistics.
  
index 539608b16d32aacb665e49f3016100ebf3f16b03,beec1839008a8c646fd1db4d86ef11e69dbc9dd7..1652e8b37d609a19cd5d19588a00f14e405a0cec
@@@ -528,9 -528,32 +528,32 @@@ public
      ///   "expired-reclaimed" or removing it from the lease databse,
      /// - updating statistics of assigned and reclaimed leases
      ///
+     /// Note: declined leases fall under the same expiration/reclaimation
+     /// processing as normal leases. In principle, it would more elegant
+     /// to have a separate processing for declined leases reclaimation. However,
+     /// due to performance reasons we decided to use them together. Several
+     /// aspects were taken into consideration. First, normal leases are expected
+     /// to expire frequently, so in a typical deployment this method will have
+     /// some leases to process. Second, declined leases are expected to be very
+     /// rare event, so in most cases there won't be any declined expired leases.
+     /// Third, the calls to LeaseMgr to obtain all leases of specific expiration
+     /// criteria are expensive, so it is better to have one call rather than
+     /// two, especially if one of those calls is expected to usually return no
+     /// leases.
+     ///
+     /// It doesn't make sense to retain declined leases that are reclaimed,
+     /// because those leases don't contain any useful information (all client
+     /// identifying information was stripped when the leave was moved to the
+     /// declined state). Therefore remove_leases parameter is ignored for
+     /// declined leases. They are always removed.
+     ///
+     /// Also, for delined leases @ref reclaimDeclined is called. It conducts
+     /// several declined specific operation (extra log entry, stats dump,
+     /// hooks).
+     ///
      /// @param max_leases Maximum number of leases to be reclaimed.
      /// @param timeout Maximum amount of time that the reclaimation routine
 -    /// may be processing expired leases, expressed in seconds.
 +    /// may be processing expired leases, expressed in milliseconds.
      /// @param remove_lease A boolean value indicating if the lease should
      /// be removed when it is reclaimed (if true) or it should be left in the
      /// database in the "expired-reclaimed" state (if false).
Simple merge
index 2210fb791ed291f1a93e62e814fb9552148f975d,55ab33e26c434e5755b92583fc396ed84b545a17..da504e4a53804d330df18de9a44cae01101c95a1
@@@ -810,108 -727,119 +803,221 @@@ public
          EXPECT_TRUE(testLeases(&dnsUpdateGeneratedForLease, &oddLeaseIndex));
      }
  
 +    /// @brief This test verfies that callouts are executed for each expired
 +    /// lease when installed.
 +    void testReclaimExpiredLeasesHooks() {
 +        for (int i = 0; i < TEST_LEASES_NUM; ++i) {
 +            if (evenLeaseIndex(i)) {
 +                expire(i, 1000 - i);
 +            }
 +        }
 +
 +        vector<string> libraries; // no libraries at this time
 +        HooksManager::loadLibraries(libraries);
 +
 +        // Install a callout: lease4_expire or lease6_expire.
 +        std::ostringstream callout_name;
 +        callout_name << callout_argument_name << "_expire";
 +        EXPECT_NO_THROW(HooksManager::preCalloutsLibraryHandle().registerCallout(
 +                        callout_name.str(), leaseExpireCallout));
 +
 +        ASSERT_NO_THROW(reclaimExpiredLeases(0, 0, false));
 +
 +        // Callouts should be executed for leases with even indexes and these
 +        // leases should be reclaimed.
 +        EXPECT_TRUE(testLeases(&leaseCalloutExecuted, &evenLeaseIndex));
 +        EXPECT_TRUE(testLeases(&leaseReclaimed, &evenLeaseIndex));
 +        // Callouts should not be executed for leases with odd indexes and these
 +        // leases should not be reclaimed.
 +        EXPECT_TRUE(testLeases(&leaseCalloutNotExecuted, &oddLeaseIndex));
 +        EXPECT_TRUE(testLeases(&leaseNotReclaimed, &oddLeaseIndex));
 +    }
 +
 +    /// @brief This test verfies that callouts are executed for each expired
 +    /// lease and that the lease is not reclaimed when skip flag is set.
 +    void testReclaimExpiredLeasesHooksWithSkip() {
 +        for (int i = 0; i < TEST_LEASES_NUM; ++i) {
 +            if (evenLeaseIndex(i)) {
 +                expire(i, 1000 - i);
 +            }
 +        }
 +
 +        vector<string> libraries; // no libraries at this time
 +        HooksManager::loadLibraries(libraries);
 +
 +        // Install a callout: lease4_expire or lease6_expire.
 +        std::ostringstream callout_name;
 +        callout_name << callout_argument_name << "_expire";
 +        EXPECT_NO_THROW(HooksManager::preCalloutsLibraryHandle().registerCallout(
 +                        callout_name.str(), leaseExpireWithSkipCallout));
 +
 +        ASSERT_NO_THROW(reclaimExpiredLeases(0, 0, false));
 +
 +        // Callouts should have been executed for leases with even indexes.
 +        EXPECT_TRUE(testLeases(&leaseCalloutExecuted, &evenLeaseIndex));
 +        // Callouts should not be executed for leases with odd indexes.
 +        EXPECT_TRUE(testLeases(&leaseCalloutNotExecuted, &oddLeaseIndex));
 +        // Leases shouldn't be reclaimed because the callout sets the
 +        // skip flag for each of them.
 +        EXPECT_TRUE(testLeases(&leaseNotReclaimed, &allLeaseIndexes));
 +    }
 +
 +    /// @brief This test verifies that it is possible to set the timeout for
 +    /// the execution of the lease reclamation routine.
 +    void testReclaimExpiredLeasesTimeout(const uint16_t timeout) {
 +        // Leases are segregated from the most expired to the least expired.
 +        for (int i = 0; i < TEST_LEASES_NUM; ++i) {
 +            expire(i, 2000 - i);
 +        }
 +
 +        vector<string> libraries;
 +        HooksManager::loadLibraries(libraries);
 +
 +        // Install a callout: lease4_expire or lease6_expire. Each callout
 +        // takes at least 2ms to run (it uses usleep).
 +        std::ostringstream callout_name;
 +        callout_name << callout_argument_name << "_expire";
 +        EXPECT_NO_THROW(HooksManager::preCalloutsLibraryHandle().registerCallout(
 +                        callout_name.str(), leaseExpireWithDelayCallout));
 +
 +        // Reclaim leases with timeout.
 +        ASSERT_NO_THROW(reclaimExpiredLeases(0, timeout, false));
 +
 +        // We reclaimed at most (timeout / 2ms) leases.
 +        const uint16_t theoretical_reclaimed = static_cast<uint16_t>(timeout / 2);
 +
 +        // The actual number of leases reclaimed is likely to be lower than
 +        // the theoretical number. For low theoretical number the adjusted
 +        // number is always 1. For higher number, it will be 10 less than the
 +        // theoretical number.
 +        const uint16_t adjusted_reclaimed = (theoretical_reclaimed > 10 ?
 +                                             theoretical_reclaimed - 10 : 1);
 +
 +        EXPECT_TRUE(testLeases(&leaseCalloutExecuted, &allLeaseIndexes,
 +                               LowerBound(0), UpperBound(adjusted_reclaimed)));
 +        EXPECT_TRUE(testLeases(&leaseReclaimed, &allLeaseIndexes,
 +                               LowerBound(0), UpperBound(adjusted_reclaimed)));
 +        EXPECT_TRUE(testLeases(&leaseCalloutNotExecuted, &allLeaseIndexes,
 +                               LowerBound(theoretical_reclaimed + 1),
 +                               UpperBound(TEST_LEASES_NUM)));
 +        EXPECT_TRUE(testLeases(&leaseNotReclaimed, &allLeaseIndexes,
 +                               LowerBound(theoretical_reclaimed + 1),
 +                               UpperBound(TEST_LEASES_NUM)));
 +    }
 +
+     /// @brief Test that declined expired leases can be removed.
+     ///
+     /// This method allows controlling remove_leases parameter when calling
+     /// @ref AllocEngine::reclaimExpiredLeases4. This should not matter, as
+     /// the address affinity doesn't make sense for declined leases (they don't
+     /// have any useful information in them anymore), so AllocEngine should
+     /// remove them all the time.
+     ///
+     /// @param remove see description above
+     void testReclaimDeclined4(bool remove) {
+         for (unsigned int i = 0; i < TEST_LEASES_NUM; ++i) {
+             // Mark leases with even indexes as expired.
+             if (evenLeaseIndex(i)) {
+                 // Mark lease as declined with 100 seconds of probation-period
+                 // (i.e. lease is supposed to be off limits for 100 seconds)
+                 decline(i, 100);
+                 // The higher the index, the more expired the lease.
+                 expire(i, 10 + i);
+             }
+         }
+         // Run leases reclamation routine on all leases. This should result
+         // in removing all leases with status = declined, i.e. all
+         // even leases should be gone.
+         ASSERT_NO_THROW(reclaimExpiredLeases(0, 0, remove));
+         // Leases with even indexes should not exist in the DB
+         EXPECT_TRUE(testLeases(&leaseDoesntExist, &evenLeaseIndex));
+     }
+     /// @brief Test that appropriate statistics are updated when
+     ///        declined expired leases are processed by AllocEngine.
+     void testReclaimDeclined4Stats() {
+         // Leases by default all belong to subnet_id_ = 1. Let's count the
+         // number of declined leases.
+         int subnet1_cnt = 0;
+         int subnet2_cnt = 0;
+         // Let's move all leases to declined,expired state.
+         for (unsigned int i = 0; i < TEST_LEASES_NUM; ++i) {
+             // Move the lease to declined state
+             decline(i, 100);
+             // And expire it, so it will be reclaimed
+             expire(i, 10 + 1);
+             // Move every other lease to subnet-id = 2.
+             if (evenLeaseIndex(i)) {
+                 subnet1_cnt++;
+             } else {
+                 subnet2_cnt++;
+                 setSubnetId(i, 2);
+             }
+         }
+         StatsMgr& stats_mgr = StatsMgr::instance();
+         // Let's set the global statistic. Values are arbitrary and can
+         // be used to easily detect whether a given stat was decreased or
+         // increased. They are sufficiently high compared to number of leases
+         // to avoid any chances of going into negative.
+         stats_mgr.setValue("declined-addresses", static_cast<int64_t>(1000));
+         // Let's set global the counter for reclaimed declined addresses.
+         stats_mgr.setValue("reclaimed-declined-addresses",
+                            static_cast<int64_t>(2000));
+         // And those subnet specific as well
+         stats_mgr.setValue(stats_mgr.generateName("subnet", 1,
+                            "assigned-addresses"), int64_t(1000));
+         stats_mgr.setValue(stats_mgr.generateName("subnet", 2,
+                            "assigned-addresses"), int64_t(2000));
+         stats_mgr.setValue(stats_mgr.generateName("subnet", 1,
+                            "reclaimed-declined-addresses"), int64_t(3000));
+         stats_mgr.setValue(stats_mgr.generateName("subnet", 2,
+                            "reclaimed-declined-addresses"), int64_t(4000));
+         stats_mgr.setValue(stats_mgr.generateName("subnet", 1,
+                            "declined-addresses"), int64_t(10));
+         stats_mgr.setValue(stats_mgr.generateName("subnet", 2,
+                            "declined-addresses"), int64_t(20));
+         // Run leases reclamation routine on all leases. This should result
+         // in removal of all leases with even indexes.
+         ASSERT_NO_THROW(reclaimExpiredLeases(0, 0, true));
+         // Declined-addresses should be decreased from its initial value (1000)
+         // for both recovered addresses from subnet1 and subnet2.
+         testStatistics("declined-addresses", 1000 - subnet1_cnt - subnet2_cnt);
+         // The code should bump up global counter for reclaimed declined
+         // addresses.
+         testStatistics("reclaimed-declined-addresses", 2000 + subnet1_cnt + subnet2_cnt);
+         // subnet[X].assigned-addresses should go down. Between the time
+         // of DHCPDECLINE reception and declined expired lease reclaimation,
+         // we count this address as assigned-addresses. We decrease assigned-
+         // addresses when we reclaim the lease, not when the packet is received.
+         // For explanation, see Duplicate Addresses (DHCPDECLINE support)
+         // section in the User's Guide or a comment in Dhcpv4Srv::declineLease.
+         testStatistics("subnet[1].assigned-addresses", 1000 - subnet1_cnt);
+         testStatistics("subnet[2].assigned-addresses", 2000 - subnet2_cnt);
+         // subnet[X].reclaimed-declined-addresses should go up in each subnet
+         testStatistics("subnet[1].reclaimed-declined-addresses", 3000 + subnet1_cnt);
+         testStatistics("subnet[2].reclaimed-declined-addresses", 4000 + subnet1_cnt);
+     }
      /// @brief Collection of leases created at construction time.
      std::vector<LeasePtrType> leases_;
  
@@@ -1509,36 -1404,25 +1615,57 @@@ TEST_F(ExpirationAllocEngine4Test, recl
      testReclaimExpiredLeasesStats();
  }
  
 +// This test verifies that callouts are executed for each expired lease.
 +TEST_F(ExpirationAllocEngine4Test, reclaimExpiredLeasesHooks) {
 +    testReclaimExpiredLeasesHooks();
 +}
 +
 +// This test verifies that callouts are executed for each expired lease
 +// and that the lease is not reclaimed when the skip flag is set.
 +TEST_F(ExpirationAllocEngine4Test, reclaimExpiredLeasesHooksWithSkip) {
 +    testReclaimExpiredLeasesHooksWithSkip();
 +}
 +
 +// This test verifies that it is possible to set the timeout for the
 +// execution of the lease reclamation routine.
 +TEST_F(ExpirationAllocEngine4Test, reclaimExpiredLeasesTimeout) {
 +    // This test needs at least 40 leases to make sense.
 +    BOOST_STATIC_ASSERT(TEST_LEASES_NUM >= 40);
 +    // Run with timeout of 60ms.
 +    testReclaimExpiredLeasesTimeout(60);
 +}
 +
 +// This test verifies that at least one lease is reclaimed if the timeout
 +// for the lease reclamation routine is shorter than the time needed for
 +// the reclamation of a single lease. This prevents the situation when
 +// very short timeout (perhaps misconfigured) effectively precludes leases
 +// reclamation.
 +TEST_F(ExpirationAllocEngine4Test, reclaimExpiredLeasesShortTimeout) {
 +    // We will most likely reclaim just one lease, so 5 is more than enough.
 +    BOOST_STATIC_ASSERT(TEST_LEASES_NUM >= 5);
 +    // Reclaim leases with the 1ms timeout.
 +    testReclaimExpiredLeasesTimeout(1);
 +}
 +
+ /// This test verifies that @ref AllocEngine::reclaimExpiredLeases4 properly
+ /// handles declined leases that have expired in case when it is told to
+ /// remove leases.
+ TEST_F(ExpirationAllocEngine4Test, reclaimDeclined1) {
+     testReclaimDeclined4(true);
+ }
+ /// This test verifies that @ref AllocEngine::reclaimExpiredLeases4 properly
+ /// handles declined leases that have expired in case when it is told to
+ /// not remove leases. This flag should not matter and declined expired
+ /// leases should always be removed.
+ TEST_F(ExpirationAllocEngine4Test, reclaimDeclined2) {
+     testReclaimDeclined4(false);
+ }
+ /// This test verifies that statistics are modified correctly after
+ /// reclaim expired leases is called.
+ TEST_F(ExpirationAllocEngine4Test, reclaimDeclinedStats) {
+     testReclaimDeclined4Stats();
+ }
  }; // end of anonymous namespace