--- /dev/null
+#!/usr/bin/perl -Tw
+
+######################################################################
+#
+# Copyright (C) 2020 Network RADIUS
+#
+# $Id$
+#
+######################################################################
+#
+# Helper script for generating the SQL commands to align the SQL IP pools in a
+# database with a given specification.
+#
+# The radippool table is updated is a way that preserves existing leases,
+# provided that the corresponding IP addresses still exist in their pool.
+#
+# This script typically receives the output of the generate_pool_addresses.pl
+# script, as follows:
+#
+# generate_pool_addresses.pl <options> | align_sqlippools.pl <sql_dialect>
+#
+# For example:
+#
+# generate_pool_addresses.pl main_pool 10.0.1.0 10.0.1.255 | \
+# align_sqlippools.pl mariadb
+#
+# generate_pool_addresses.pl yaml pool_defs.yml existing_ips.txt | \
+# align_sqlippools.pl postgresql
+#
+# For the latter example the existing_ips.txt file might be created as
+# follows:
+#
+# psql radius -qtAc 'SELECT framedipaddress FROM radippool' > existing_ips.txt
+#
+# Note: The generate_pool_addresses.pl script describes the input format
+# expected by this script (as well as the format of the pool_defs.yml and
+# existing_ips.txt files.)
+#
+# Output:
+#
+# The output of this script is the SQL command sequence for aligning the pools
+# with the definition provided, which you should review before running them
+# against your database.
+#
+
+use strict;
+use Template;
+
+my %template=load_templates();
+
+if ($#ARGV != 0) {
+ print STDERR <<'EOF';
+Usage: generate_pool_addresses.pl ... | align_sqlippools.pl <dialect>
+
+EOF
+ exit 1;
+}
+
+my $dialect=$ARGV[0];
+
+unless (defined $template{$dialect}) {
+ print STDERR "Unknown dialect. Pick one of: ";
+ print STDERR "$_ " foreach sort keys %template;
+ print STDERR "\n";
+ exit 1;
+}
+
+my @ips=();
+
+my $line = 0;
+while (<STDIN>) {
+ $line++;
+
+ chomp;
+
+ next if $_ =~ /^#/ || $_ =~ /^\s*$/;
+
+ # The SQL works out what addresses to remove by itself
+ next if $_ =~ /^-/;
+
+ (my $action, my $pool_name, my $ip) = $_ =~ /^(.)\s+(.+)\s+([^\s]+)$/;
+
+ unless (defined $ip) {
+ warn "Unrecognised line $line: $_";
+ next;
+ }
+
+ push @ips, { poolname => $pool_name, ip => $ip };
+
+}
+
+my $tt=Template->new();
+$tt->process(\$template{$dialect}, {ips => \@ips, batchsize => 100}) || die $tt->error();
+
+exit 0;
+
+
+#
+# SQL dialect templates
+#
+
+sub load_templates {
+
+ my %template;
+
+#
+# MariaDB
+#
+ $template{'mariadb'} = <<'END_mariadb';
+-- Temporary table holds the provided IP pools
+DROP TEMPORARY TABLE IF EXISTS radippool_temp;
+CREATE TEMPORARY TABLE radippool_temp (
+ id int(11) unsigned NOT NULL auto_increment,
+ pool_name varchar(30) NOT NULL,
+ framedipaddress varchar(15) NOT NULL,
+ PRIMARY KEY (id),
+ KEY pool_name_framedipaddress (pool_name,framedipaddress)
+);
+
+-- Populate the temporary table
+[%- FOREACH m IN ips %]
+[%- "\n\nINSERT INTO radippool_temp (pool_name,framedipaddress) VALUES" IF loop.index % batchsize == 0 %]
+[%- IF (loop.index+1) % batchsize == 0 OR loop.last %]
+('[% m.poolname %]','[% m.ip %]');
+[%- ELSE %]
+('[% m.poolname %]','[% m.ip %]'),
+[%- END %]
+[%- END %]
+
+START TRANSACTION;
+
+-- Delete old pools that have been removed
+DELETE r FROM radippool r
+ LEFT JOIN radippool_temp t USING (pool_name,framedipaddress)
+ WHERE t.id IS NULL;
+
+-- Add new pools that have been created
+INSERT INTO radippool (pool_name,framedipaddress)
+ SELECT pool_name,framedipaddress FROM radippool_temp t WHERE NOT EXISTS (
+ SELECT * FROM radippool r
+ WHERE r.pool_name=t.pool_name AND r.framedipaddress=t.framedipaddress
+ );
+
+COMMIT;
+END_mariadb
+
+
+#
+# PostgreSQL
+#
+ $template{'postgresql'} = <<'END_postgresql';
+-- Temporary table holds the provided IP pools
+DROP TABLE IF EXISTS radippool_temp;
+CREATE TEMPORARY TABLE radippool_temp (
+ pool_name varchar(64) NOT NULL,
+ FramedIPAddress INET NOT NULL
+);
+CREATE INDEX radippool_temp_idx ON radippool_temp USING btree (pool_name,FramedIPAddress);
+
+-- Populate the temporary table
+[%- FOREACH m IN ips %]
+[%- "\n\nINSERT INTO radippool_temp (pool_name,framedipaddress) VALUES" IF loop.index % batchsize == 0 %]
+[%- IF (loop.index+1) % batchsize == 0 OR loop.last %]
+('[% m.poolname %]','[% m.ip %]');
+[%- ELSE %]
+('[% m.poolname %]','[% m.ip %]'),
+[%- END %]
+[%- END %]
+
+START TRANSACTION;
+
+-- Delete old pools that have been removed
+DELETE FROM radippool r WHERE NOT EXISTS (
+ SELECT FROM radippool_temp t
+ WHERE t.pool_name = r.pool_name AND t.framedipaddress = r.framedipaddress
+);
+
+-- Add new pools that have been created
+INSERT INTO radippool (pool_name,framedipaddress)
+ SELECT pool_name,framedipaddress FROM radippool_temp t WHERE NOT EXISTS (
+ SELECT * FROM radippool r
+ WHERE r.pool_name=t.pool_name AND r.framedipaddress=t.framedipaddress
+ );
+
+COMMIT;
+END_postgresql
+
+ return %template;
+
+}
--- /dev/null
+#!/usr/bin/perl -Tw
+
+######################################################################
+#
+# Copyright (C) 2020 Network RADIUS
+#
+# $Id$
+#
+######################################################################
+#
+# Helper script for populating IP pools with address entries.
+#
+# This script generates output that is useful for populating an IP pool for
+# use with FreeRADIUS (and possibly other purposes). The pool may be
+# implemented as an SQL IP Pool (rlm_sqlippool) or any other backing store
+# that has one entry per IP address.
+#
+# This script output a list of address to add, retain and remove in order to
+# align a pool to a specification. It is likely that you will want to
+# process the output to generate the actual commands (e.g. SQL statements)
+# that make changes to the datastore. For example:
+#
+# generate_pool_addresses.pl ... | align_sql_pools.pl postgresql
+#
+#
+# Use with a single address range
+# -------------------------------
+#
+# For basic use, arguments can be provided to this script that denote the ends
+# of a single IP (v4 or v6) address range together with the pool_name.
+#
+# Optionally the number of IPs to sparsely populate the range with can be
+# provided. If the range is wider than a /16 then the population of the range
+# is capped at 65536 IPs, unless otherwise specified.
+#
+# In the case that a sparse range is defined, a file containing pre-existing
+# IP entries can be provided. The range will be populated with entries from
+# this file that fall within the range, prior to the remainder of the range
+# being populated with random address in the range.
+#
+# generate_pool_addresses.pl <pool_name> <range_start> <range_end> \
+# [ <capacity> [ <existing_ips_file> ] ]
+#
+# Note: Sparse ranges are populated using a deterministic, pseudo-random
+# function. This allows pools to be trivially extended without having to
+# supply the existing contents using a file. If you require
+# less-predictable randomness or a different random sequence then remove
+# or modify the line calling srand(), below.
+#
+#
+# Use with multiple pools and address ranges
+# ------------------------------------------
+#
+# For more complex us, the script allows a set of pool definitions to be
+# provided in a YAML file which describes a set of one or more pools, each
+# containing a set of one or more ranges. The first argument in this case is
+# always "yaml":
+#
+# generate_pool_addresses.pl yaml <pool_defs_yaml_file> [ <existing_ips_file> ]
+#
+# The format for the YAML file is demonstrated by the following example:
+#
+# pool_with_a_single_contiguous_range:
+# - start: 192.0.2.3
+# end: 192.0.2.250
+#
+# pool_with_a_single_sparse_range:
+# - start: 10.10.10.0
+# end: 10.10.20.255
+# capacity: 200
+#
+# pool_with_multiple_ranges:
+# - start: 10.10.10.1
+# end: 10.10.10.253
+# - start: 10.10.100.0
+# end: 10.10.199.255
+# capacity: 1000
+#
+# v6_pool_with_contiguous_range:
+# - start: '2001:db8:1:2:3:4:5:10'
+# end: '2001:db8:1:2:3:4:5:7f'
+#
+# v6_pool_with_sparse_range:
+# - start: '2001:db8:1:2::'
+# end: '2001:db8:1:2:ffff:ffff:ffff:ffff'
+# capacity: 200
+#
+# As with the basic use case, a file containing pre-existing IP entries can be
+# provided with which any sparse ranges will be populated ahead of any random
+# addresses.
+#
+#
+# Output
+# ------
+#
+# The script returns line-based output beginning with "+", "=" or "-", and
+# includes the pool_name and an IP address.
+#
+# + pool_name 192.0.2.10
+#
+# A new address to be added to the corresponding range in the pool.
+#
+# = pool_name 192.0.2.20
+#
+# A pre-existing address that is to be retained in the pool. (Only if a
+# pre-existing pool entries file is given.)
+#
+# - pool_name 192.0.2.30
+#
+# A pre-existing address that is to be removed from the corresponding
+# range in the pool. (Only if a pre-existing pool entries file is given.)
+#
+# # main_pool: 192.0.10.3 - 192.0.12.250 (500)
+#
+# Lines beginning with "#" are comments
+#
+#
+# Examples
+# --------
+#
+# generate_pool_addresses.pl main_pool 192.0.2.3 192.0.2.249
+#
+# Will create a pool from a full populated IPv4 range, i.e. all IPs in the
+# range available for allocation).
+#
+# generate_pool_addresses.pl main_pool 10.66.0.0 10.66.255.255 10000
+#
+# Will create a pool from a sparsely populated IPv4 range for a /16
+# network (maximum of 65.536 addresses), populating the range with 10,000
+# addreses. The effective size of the pool can be increased in future by
+# increasing the capacity of the range with:
+#
+# generate_pool_addresses.pl main_pool 10.66.0.0 10.66.255.255 20000
+#
+# This generates the same initial set of 10,000 addresses as the previous
+# example but will create 20,000 addresses overall, unless the random seed
+# has been amended since the initial run.
+#
+# generate_pool_addresses.pl main_pool 2001:db8:1:2:: \
+# 2001:db8:1:2:ffff:ffff:ffff:ffff
+#
+# Will create a pool from the IPv6 range 2001:db8:1:2::/64, initially
+# populating the range with 65536 (by default) addresses.
+#
+# generate_pool_addresses.pl main_pool 2001:db8:1:2:: \
+# 2001:db8:1:2:ffff:ffff:ffff:ffff \
+# 10000 existing_ips.txt
+#
+# Will create a pool using the same range as the previous example, but
+# this time the range will be populated with 10,000 addresses. The range
+# will be populated using lines extracted from the `existing_ips.txt` file
+# that represent IPs which fall within range.
+#
+# generate_pool_addresses.pl yaml pool_defs.yml existing_ips.txt
+#
+# Will create one of more pools using the definitions found in the
+# pool_defs.yml YAML file. The pools will contain one or more ranges with
+# each of the ranges first being populated with entries from the
+# existing_ips.txt file that fall within the range, before being filled
+# with random addresses to the defined capacity.
+#
+
+use strict;
+use Net::IP qw/ip_bintoip ip_iptobin ip_bincomp ip_binadd ip_is_ipv4 ip_is_ipv6/;
+
+my $yaml_available = 0;
+
+eval {
+ require YAML::XS;
+ YAML::XS->import('LoadFile');
+ $yaml_available = 1;
+};
+
+if ($#ARGV < 2 || $#ARGV > 4) {
+ usage();
+ exit 1;
+}
+
+if ($ARGV[0] eq 'yaml') {
+
+ if ($#ARGV > 3) {
+ usage();
+ exit 1;
+ }
+
+ unless ($yaml_available) {
+ die "ERROR: YAML is not available. Install the YAML::XS Perl module.";
+ }
+ process_yaml_file();
+
+ goto done;
+
+}
+
+process_commandline();
+
+done:
+
+exit 0;
+
+
+sub usage {
+ print STDERR <<'EOF'
+Usage:
+ generate_pool_addresses.pl <pool_name> <range_start> <range_end> [ <capacity> [ <existing_ips_file> ] ]
+or:
+ generate_pool_addresses.pl yaml <pool_defs_yaml_file> [ <existing_ips_file> ]
+
+EOF
+}
+
+
+sub process_commandline {
+
+ $SIG{__DIE__} = sub { usage(); die(@_); };
+
+ my $pool_name = $ARGV[0];
+ my $range_start = $ARGV[1];
+ my $range_end = $ARGV[2];
+ my $capacity = $ARGV[3];
+
+ my @entries = ();
+ @entries = load_entries($ARGV[4]) if ($#ARGV >= 4);
+
+ handle_range($pool_name, $range_start, $range_end, $capacity, @entries);
+
+}
+
+
+sub process_yaml_file {
+
+ my $yaml_file = $ARGV[1];
+
+ unless (-r $yaml_file) {
+ die "ERROR: Cannot open <pool_defs_yaml_file> for reading: $yaml_file";
+ }
+
+ my %pool_defs = %{LoadFile($yaml_file)};
+
+ my @entries = ();
+ @entries = load_entries($ARGV[2]) if ($#ARGV >= 2);
+
+ foreach my $pool_name (sort keys %pool_defs) {
+ foreach my $range (@{$pool_defs{$pool_name}}) {
+ my $range_start = $range->{start};
+ my $range_end = $range->{end};
+ my $capacity = $range->{capacity};
+ handle_range($pool_name, $range_start, $range_end, $capacity, @entries);
+ }
+ }
+
+}
+
+
+sub load_entries {
+
+ my $entries_file = shift;
+
+ my @entries = ();
+ unless (-r $entries_file) {
+ die "ERROR: Cannot open <existing_ips_file> for reading: $entries_file"
+ }
+ open(my $fh, "<", $entries_file) || die "Failed to open $entries_file";
+ while(<$fh>) {
+ chomp;
+ push @entries, $_;
+ }
+
+ return @entries;
+
+}
+
+
+sub handle_range {
+
+ my $pool_name = shift;
+ my $range_start = shift;
+ my $range_end = shift;
+ my $capacity = shift;
+ my @entries = @_;
+
+ unless (ip_is_ipv4($range_start) || ip_is_ipv6($range_start)) {
+ die "ERROR: Incorrectly formatted IPv4/IPv6 address for range_start: $range_start";
+ }
+
+ unless (ip_is_ipv4($range_end) || ip_is_ipv6($range_end)) {
+ die "ERROR: Incorrectly formatted IPv4/IPv6 address for range_end: $range_end";
+ }
+
+ my $ip_start = new Net::IP($range_start);
+ my $ip_end = new Net::IP($range_end);
+ my $ip_range = new Net::IP("$range_start - $range_end");
+
+ unless (defined $ip_range) {
+ die "ERROR: The range defined by <range_start> - <range_end> is invalid: $range_start - $range_end";
+ }
+
+ my $range_size = $ip_range->size;
+ $capacity = $range_size < 65536 ? "$range_size" : 65536 unless defined $capacity;
+
+ if ($range_size < $capacity) {
+ $capacity = "$range_size";
+ warn "WARNING: Insufficent IPs in the range. Will create $capacity entries.";
+ }
+
+ # Prune the entries to only those within the specified range
+ for (my $i = 0; $i <= $#entries; $i++) {
+ my $version = ip_is_ipv4($entries[$i]) ? 4 : 6;
+ my $binip = ip_iptobin($entries[$i],$version);
+ if ($ip_start->version != $version ||
+ ip_bincomp($binip, 'lt', $ip_start->binip) == 1 ||
+ ip_bincomp($binip, 'gt', $ip_end->binip) == 1) {
+ $entries[$i]='';
+ }
+ }
+
+ #
+ # We use the sparse method if the number of entries available occupies < 80% of
+ # the network range, otherwise we use a method that involves walking the
+ # entire range.
+ #
+
+ srand(42); # Set the seed for the PRNG
+
+ if (length($range_size) > 9 || $capacity / "$range_size" < 0.8) { # From "BigInt" to FP
+ @entries = sparse_fill($pool_name, $ip_start, $ip_end, $capacity, @entries);
+ } else {
+ @entries = dense_fill($pool_name, $ip_start, $ip_end, $ip_range, $capacity, @entries);
+ }
+
+ print "# $pool_name: $range_start - $range_end ($capacity)\n";
+ print "$_\n" foreach @entries;
+ print "\n";
+
+}
+
+
+#
+# With this sparse fill method we randomly allocate within the scope of the
+# smallest enclosing network prefix, checking that we are within the given
+# range, retrying if we are outside or we hit a duplicate.
+#
+# This method can efficiently choose a small number of addresses relative to
+# the size of the range. It becomes slower as the population of a range nears
+# the range's limit since it is harder to choose a free address at random.
+#
+# It is useful for selecting a handful of addresses from an enourmous IPv6 /64
+# network for example.
+#
+sub sparse_fill {
+
+ my $pool_name = shift;
+ my $ip_start = shift;
+ my $ip_end = shift;
+ my $capacity = shift;
+ my @entries = @_;
+
+ # Find the smallest network that encloses the given range
+ my $version = $ip_start->version;
+ ( $ip_start->binip ^ $ip_end->binip ) =~ /^\0*/;
+ my $net_prefix = $+[0];
+ my $net_bits = substr($ip_start->binip, 0, $net_prefix);
+ my $host_length = length($ip_start->binip) - $net_prefix;
+
+ my %ips = ();
+ my $i = 0;
+ while ($i < $capacity) {
+
+ # Use the given entries first
+ my $rand_ip;
+ my $given_lease = 0;
+ shift @entries while $#entries >= 0 && $entries[0] eq '';
+ if ($#entries >= 0) {
+ $rand_ip = ip_iptobin(shift @entries, $version);
+ $given_lease = 1;
+ } else {
+ $rand_ip = $net_bits;
+ $rand_ip .= [0..1]->[rand 2] for 1..$host_length;
+ # Check that we are inside the given range
+ next if ip_bincomp($rand_ip, 'lt', $ip_start->binip) == 1 ||
+ ip_bincomp($rand_ip, 'gt', $ip_end->binip) == 1;
+ }
+
+ next if defined $ips{$rand_ip};
+
+ $ips{$rand_ip} = $given_lease ? '=' : '+';
+ $i++;
+
+ }
+
+ # Allow the pool to be shrunk
+ $ips{ip_iptobin($_, $version)} = '-' foreach @entries;
+
+ return map { $ips{$_}." ".$pool_name." ".ip_bintoip($_, $version) } sort keys %ips;
+
+}
+
+
+#
+# With this dense fill method, after first selecting the given entries we walk
+# the network range picking IPs with evenly distributed probability.
+#
+# This method can efficiently choose a large number of addresses relative to
+# the size of a range, provided that the range isn't massive. It becomes
+# slower as the range size increases.
+#
+sub dense_fill {
+
+ my $pool_name = shift;
+ my $ip_start = shift;
+ my $ip_end = shift;
+ my $ip_range = shift;
+ my $capacity = shift;
+ my @entries = @_;
+
+ my $version = $ip_start->version;
+
+ my $one = ("0"x($version == 4 ? 31 : 127)) . '1';
+
+ my %ips = ();
+ my $remaining_entries = $capacity;
+ my $remaining_ips = $ip_range->size;
+ my $ipbin = $ip_start->binip;
+
+ while ($remaining_entries > 0 && (ip_bincomp($ipbin, 'le', $ip_end->binip) == 1)) {
+
+ # Use the given entries first
+ shift @entries while $#entries >= 0 && $entries[0] eq '';
+ if ($#entries >= 0) {
+ $ips{ip_iptobin(shift @entries, $version)} = '=';
+ $remaining_entries--;
+ $remaining_ips--;
+ next;
+ }
+
+ goto next_ip if defined $ips{$ipbin};
+
+ # Skip the IP that we have already selected by given entries, otherwise
+ # randomly pick it
+ if (!defined $ips{$ipbin} &&
+ (rand) <= $remaining_entries / "$remaining_ips") { # From "BigInt" to FP
+ $ips{$ipbin} = '+';
+ $remaining_entries--;
+ }
+
+ $remaining_ips--;
+ $ipbin = ip_binadd($ipbin,$one);
+
+ }
+
+ # Allow the pool to be shrunk
+ $ips{ip_iptobin($_, $version)} = '-' foreach @entries;
+
+ return map { $ips{$_}." ".$pool_name." ".ip_bintoip($_, $version) } sort keys %ips;
+
+}
projects / thirdparty / freeradius-server.git / commitdiff
