]> git.ipfire.org Git - people/pmueller/ipfire-2.x.git/blob - src/patches/libloc-0.9.4-upstream.patch
projects / people / pmueller / ipfire-2.x.git / blob
? search:


28d569d93d2b64bc15021cd258b6e50bb1e6bff5
[people/pmueller/ipfire-2.x.git] / src / patches / libloc-0.9.4-upstream.patch
1 From ee6ea3986dc80183157f67275dc9f28231b5d5b2 Mon Sep 17 00:00:00 2001
2 From: Michael Tremer <michael.tremer@ipfire.org>
3 Date: Thu, 24 Sep 2020 10:17:58 +0000
4 Subject: [PATCH 01/70] Revert "importer: Purge any redundant entries"
5
6 This reverts commit c2cc55d5a6875c3838f060032eaed89dcfb92ef6.
7
8 The query stalls the database and therefore the automatic
9 scripts are no longer able to generate a new version of the
10 database.
11
12 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
13 ---
14 src/python/location-importer.in | 22 +---------------------
15 1 file changed, 1 insertion(+), 21 deletions(-)
16
17 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
18 index 1467923..e3a07a0 100644
19 --- a/src/python/location-importer.in
20 +++ b/src/python/location-importer.in
21 @@ -374,27 +374,7 @@ class CLI(object):
22 INSERT INTO autnums(number, name)
23 SELECT _autnums.number, _organizations.name FROM _autnums
24 JOIN _organizations ON _autnums.organization = _organizations.handle
25 - ON CONFLICT (number) DO UPDATE SET name = excluded.name
26 - """)
27 -
28 - self.db.execute("""
29 - --- Purge any redundant entries
30 - CREATE TEMPORARY TABLE _garbage ON COMMIT DROP
31 - AS
32 - SELECT network FROM networks candidates
33 - WHERE EXISTS (
34 - SELECT FROM networks
35 - WHERE
36 - networks.network << candidates.network
37 - AND
38 - networks.country = candidates.country
39 - );
40 -
41 - CREATE UNIQUE INDEX _garbage_search ON _garbage USING BTREE(network);
42 -
43 - DELETE FROM networks WHERE EXISTS (
44 - SELECT FROM _garbage WHERE networks.network = _garbage.network
45 - );
46 + ON CONFLICT (number) DO UPDATE SET name = excluded.name;
47 """)
48
49 # Download all extended sources
50 --
51 2.20.1
52
53 From 92f6abf4e272672bb0a71cfe991261b95ebe2fef Mon Sep 17 00:00:00 2001
54 From: Michael Tremer <michael.tremer@ipfire.org>
55 Date: Thu, 24 Sep 2020 10:18:58 +0000
56 Subject: [PATCH 02/70] Revert "importer: Import raw sources for inetnum's
57 again"
58
59 This reverts commit 64e95fa903edec8b4e4e59830b395e2e4a411853.
60
61 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
62 ---
63 src/python/importer.py | 14 ++++----
64 src/python/location-importer.in | 63 ---------------------------------
65 2 files changed, 7 insertions(+), 70 deletions(-)
66
67 diff --git a/src/python/importer.py b/src/python/importer.py
68 index f19db4b..de20f37 100644
69 --- a/src/python/importer.py
70 +++ b/src/python/importer.py
71 @@ -30,8 +30,8 @@ WHOIS_SOURCES = (
72 "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
73
74 # Asia Pacific Network Information Centre
75 - "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
76 - "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
77 + #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
78 + #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
79 #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
80 #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
81 "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
82 @@ -45,8 +45,8 @@ WHOIS_SOURCES = (
83 # XXX ???
84
85 # Réseaux IP Européens
86 - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
87 - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
88 + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
89 + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
90 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
91 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
92 "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
93 @@ -55,10 +55,10 @@ WHOIS_SOURCES = (
94
95 EXTENDED_SOURCES = (
96 # African Network Information Centre
97 - #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
98 + "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
99
100 # Asia Pacific Network Information Centre
101 - #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
102 + "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
103
104 # American Registry for Internet Numbers
105 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
106 @@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
107 "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
108
109 # Réseaux IP Européens
110 - #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
111 + "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
112 )
113
114 class Downloader(object):
115 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
116 index e3a07a0..77952f2 100644
117 --- a/src/python/location-importer.in
118 +++ b/src/python/location-importer.in
119 @@ -393,10 +393,6 @@ class CLI(object):
120 if line.startswith("aut-num:"):
121 return self._parse_autnum_block(block)
122
123 - # inetnum
124 - if line.startswith("inet6num:") or line.startswith("inetnum:"):
125 - return self._parse_inetnum_block(block)
126 -
127 # organisation
128 elif line.startswith("organisation:"):
129 return self._parse_org_block(block)
130 @@ -426,65 +422,6 @@ class CLI(object):
131 autnum.get("asn"), autnum.get("org"),
132 )
133
134 - def _parse_inetnum_block(self, block):
135 - logging.debug("Parsing inetnum block:")
136 -
137 - inetnum = {}
138 - for line in block:
139 - logging.debug(line)
140 -
141 - # Split line
142 - key, val = split_line(line)
143 -
144 - if key == "inetnum":
145 - start_address, delim, end_address = val.partition("-")
146 -
147 - # Strip any excess space
148 - start_address, end_address = start_address.rstrip(), end_address.strip()
149 -
150 - # Convert to IP address
151 - try:
152 - start_address = ipaddress.ip_address(start_address)
153 - end_address = ipaddress.ip_address(end_address)
154 - except ValueError:
155 - logging.warning("Could not parse line: %s" % line)
156 - return
157 -
158 - # Set prefix to default
159 - prefix = 32
160 -
161 - # Count number of addresses in this subnet
162 - num_addresses = int(end_address) - int(start_address)
163 - if num_addresses:
164 - prefix -= math.log(num_addresses, 2)
165 -
166 - inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
167 -
168 - elif key == "inet6num":
169 - inetnum[key] = val
170 -
171 - elif key == "country":
172 - if val == "UNITED STATES":
173 - val = "US"
174 -
175 - inetnum[key] = val.upper()
176 -
177 - # Skip empty objects
178 - if not inetnum:
179 - return
180 -
181 - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
182 -
183 - # Bail out in case we have processed a non-public IP network
184 - if network.is_private:
185 - logging.warning("Skipping non-globally routable network: %s" % network)
186 - return
187 -
188 - self.db.execute("INSERT INTO networks(network, country) \
189 - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
190 - "%s" % network, inetnum.get("country"),
191 - )
192 -
193 def _parse_org_block(self, block):
194 org = {}
195 for line in block:
196 --
197 2.20.1
198
199 From f532841e9197ce2f40aad8c086d786b2cb783a54 Mon Sep 17 00:00:00 2001
200 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
201 Date: Mon, 12 Oct 2020 20:53:31 +0000
202 Subject: [PATCH 03/70] Revert "Revert "importer: Import raw sources for
203 inetnum's again""
204
205 This reverts commit 92f6abf4e272672bb0a71cfe991261b95ebe2fef.
206
207 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
208 ---
209 src/python/importer.py | 14 ++++----
210 src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++
211 2 files changed, 70 insertions(+), 7 deletions(-)
212
213 diff --git a/src/python/importer.py b/src/python/importer.py
214 index de20f37..f19db4b 100644
215 --- a/src/python/importer.py
216 +++ b/src/python/importer.py
217 @@ -30,8 +30,8 @@ WHOIS_SOURCES = (
218 "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
219
220 # Asia Pacific Network Information Centre
221 - #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
222 - #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
223 + "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
224 + "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
225 #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
226 #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
227 "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
228 @@ -45,8 +45,8 @@ WHOIS_SOURCES = (
229 # XXX ???
230
231 # Réseaux IP Européens
232 - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
233 - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
234 + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
235 + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
236 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
237 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
238 "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
239 @@ -55,10 +55,10 @@ WHOIS_SOURCES = (
240
241 EXTENDED_SOURCES = (
242 # African Network Information Centre
243 - "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
244 + #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
245
246 # Asia Pacific Network Information Centre
247 - "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
248 + #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
249
250 # American Registry for Internet Numbers
251 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
252 @@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
253 "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
254
255 # Réseaux IP Européens
256 - "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
257 + #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
258 )
259
260 class Downloader(object):
261 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
262 index 77952f2..e3a07a0 100644
263 --- a/src/python/location-importer.in
264 +++ b/src/python/location-importer.in
265 @@ -393,6 +393,10 @@ class CLI(object):
266 if line.startswith("aut-num:"):
267 return self._parse_autnum_block(block)
268
269 + # inetnum
270 + if line.startswith("inet6num:") or line.startswith("inetnum:"):
271 + return self._parse_inetnum_block(block)
272 +
273 # organisation
274 elif line.startswith("organisation:"):
275 return self._parse_org_block(block)
276 @@ -422,6 +426,65 @@ class CLI(object):
277 autnum.get("asn"), autnum.get("org"),
278 )
279
280 + def _parse_inetnum_block(self, block):
281 + logging.debug("Parsing inetnum block:")
282 +
283 + inetnum = {}
284 + for line in block:
285 + logging.debug(line)
286 +
287 + # Split line
288 + key, val = split_line(line)
289 +
290 + if key == "inetnum":
291 + start_address, delim, end_address = val.partition("-")
292 +
293 + # Strip any excess space
294 + start_address, end_address = start_address.rstrip(), end_address.strip()
295 +
296 + # Convert to IP address
297 + try:
298 + start_address = ipaddress.ip_address(start_address)
299 + end_address = ipaddress.ip_address(end_address)
300 + except ValueError:
301 + logging.warning("Could not parse line: %s" % line)
302 + return
303 +
304 + # Set prefix to default
305 + prefix = 32
306 +
307 + # Count number of addresses in this subnet
308 + num_addresses = int(end_address) - int(start_address)
309 + if num_addresses:
310 + prefix -= math.log(num_addresses, 2)
311 +
312 + inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
313 +
314 + elif key == "inet6num":
315 + inetnum[key] = val
316 +
317 + elif key == "country":
318 + if val == "UNITED STATES":
319 + val = "US"
320 +
321 + inetnum[key] = val.upper()
322 +
323 + # Skip empty objects
324 + if not inetnum:
325 + return
326 +
327 + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
328 +
329 + # Bail out in case we have processed a non-public IP network
330 + if network.is_private:
331 + logging.warning("Skipping non-globally routable network: %s" % network)
332 + return
333 +
334 + self.db.execute("INSERT INTO networks(network, country) \
335 + VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
336 + "%s" % network, inetnum.get("country"),
337 + )
338 +
339 def _parse_org_block(self, block):
340 org = {}
341 for line in block:
342 --
343 2.20.1
344
345 From a36bc686865fc87ea386fd90b389338bdcb80cbc Mon Sep 17 00:00:00 2001
346 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
347 Date: Mon, 12 Oct 2020 20:53:32 +0000
348 Subject: [PATCH 04/70] location-importer.in: only import relevant data from
349 AFRINIC, APNIC and RIPE
350 MIME-Version: 1.0
351 Content-Type: text/plain; charset=UTF-8
352 Content-Transfer-Encoding: 8bit
353
354 In contrast to ARIN and LACNIC, we are able to process more detailled
355 feeds from those RIRs, avoiding storage of obviously unnecessary data.
356
357 Thanks to various SQL optimisations, doing so now takes less time than
358 the first version of this did.
359
360 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
361 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
362 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
363 ---
364 src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++-
365 1 file changed, 87 insertions(+), 2 deletions(-)
366
367 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
368 index e3a07a0..093f325 100644
369 --- a/src/python/location-importer.in
370 +++ b/src/python/location-importer.in
371 @@ -165,6 +165,7 @@ class CLI(object):
372 -- networks
373 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
374 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
375 + CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
376 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
377
378 -- overrides
379 @@ -363,6 +364,16 @@ class CLI(object):
380 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
381 ON COMMIT DROP;
382 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
383 +
384 + CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
385 + ON COMMIT DROP;
386 + CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
387 + CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
388 + """)
389 +
390 + # Remove all previously imported content
391 + self.db.execute("""
392 + TRUNCATE TABLE networks;
393 """)
394
395 for source in location.importer.WHOIS_SOURCES:
396 @@ -370,6 +381,67 @@ class CLI(object):
397 for block in f:
398 self._parse_block(block)
399
400 + # Process all parsed networks from every RIR we happen to have access to,
401 + # insert the largest network chunks into the networks table immediately...
402 + families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
403 +
404 + for family in (row.family for row in families):
405 + smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
406 +
407 + self.db.execute("INSERT INTO networks(network, country) \
408 + SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
409 +
410 + # ... determine any other prefixes for this network family, ...
411 + prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
412 + WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
413 +
414 + # ... and insert networks with this prefix in case they provide additional
415 + # information (i. e. subnet of a larger chunk with a different country)
416 + for prefix in (row.prefix for row in prefixes):
417 + self.db.execute("""
418 + WITH candidates AS (
419 + SELECT
420 + _rirdata.network,
421 + _rirdata.country
422 + FROM
423 + _rirdata
424 + WHERE
425 + family(_rirdata.network) = %s
426 + AND
427 + masklen(_rirdata.network) = %s
428 + ),
429 + filtered AS (
430 + SELECT
431 + DISTINCT ON (c.network)
432 + c.network,
433 + c.country,
434 + masklen(networks.network),
435 + networks.country AS parent_country
436 + FROM
437 + candidates c
438 + LEFT JOIN
439 + networks
440 + ON
441 + c.network << networks.network
442 + ORDER BY
443 + c.network,
444 + masklen(networks.network) DESC NULLS LAST
445 + )
446 + INSERT INTO
447 + networks(network, country)
448 + SELECT
449 + network,
450 + country
451 + FROM
452 + filtered
453 + WHERE
454 + parent_country IS NULL
455 + OR
456 + country <> parent_country
457 + ON CONFLICT DO NOTHING""",
458 + family, prefix,
459 + )
460 +
461 self.db.execute("""
462 INSERT INTO autnums(number, name)
463 SELECT _autnums.number, _organizations.name FROM _autnums
464 @@ -470,17 +542,30 @@ class CLI(object):
465 inetnum[key] = val.upper()
466
467 # Skip empty objects
468 - if not inetnum:
469 + if not inetnum or not "country" in inetnum:
470 return
471
472 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
473
474 + # Bail out in case we have processed a network covering the entire IP range, which
475 + # is necessary to work around faulty (?) IPv6 network processing
476 + if network.prefixlen == 0:
477 + logging.warning("Skipping network covering the entire IP adress range: %s" % network)
478 + return
479 +
480 + # Bail out in case we have processed a network whose prefix length indicates it is
481 + # not globally routable (we have decided not to process them at the moment, as they
482 + # significantly enlarge our database without providing very helpful additional information)
483 + if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
484 + logging.info("Skipping network too small to be publicly announced: %s" % network)
485 + return
486 +
487 # Bail out in case we have processed a non-public IP network
488 if network.is_private:
489 logging.warning("Skipping non-globally routable network: %s" % network)
490 return
491
492 - self.db.execute("INSERT INTO networks(network, country) \
493 + self.db.execute("INSERT INTO _rirdata(network, country) \
494 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
495 "%s" % network, inetnum.get("country"),
496 )
497 --
498 2.20.1
499
500 From 2373de384f10f5573bbd7570f5522545df70c0e3 Mon Sep 17 00:00:00 2001
501 From: Michael Tremer <michael.tremer@ipfire.org>
502 Date: Fri, 16 Oct 2020 12:24:58 +0000
503 Subject: [PATCH 05/70] location-importer: Include all overridden networks
504
505 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
506 ---
507 src/python/location-importer.in | 2 ++
508 1 file changed, 2 insertions(+)
509
510 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
511 index 093f325..d249a35 100644
512 --- a/src/python/location-importer.in
513 +++ b/src/python/location-importer.in
514 @@ -240,6 +240,8 @@ class CLI(object):
515 SELECT network FROM announcements
516 UNION
517 SELECT network FROM networks
518 + UNION
519 + SELECT network FROM network_overrides
520 ORDER BY network
521 )
522
523 --
524 2.20.1
525
526 From 13f67f285856e8eabfeff2daf1be3aeaa36a82cc Mon Sep 17 00:00:00 2001
527 From: Michael Tremer <michael.tremer@ipfire.org>
528 Date: Fri, 16 Oct 2020 12:26:38 +0000
529 Subject: [PATCH 06/70] Revert "location-importer.in: only import relevant data
530 from AFRINIC, APNIC and RIPE"
531
532 This reverts commit a36bc686865fc87ea386fd90b389338bdcb80cbc.
533
534 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
535 ---
536 src/python/location-importer.in | 89 +--------------------------------
537 1 file changed, 2 insertions(+), 87 deletions(-)
538
539 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
540 index d249a35..b220eaf 100644
541 --- a/src/python/location-importer.in
542 +++ b/src/python/location-importer.in
543 @@ -165,7 +165,6 @@ class CLI(object):
544 -- networks
545 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
546 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
547 - CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
548 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
549
550 -- overrides
551 @@ -366,16 +365,6 @@ class CLI(object):
552 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
553 ON COMMIT DROP;
554 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
555 -
556 - CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
557 - ON COMMIT DROP;
558 - CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
559 - CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
560 - """)
561 -
562 - # Remove all previously imported content
563 - self.db.execute("""
564 - TRUNCATE TABLE networks;
565 """)
566
567 for source in location.importer.WHOIS_SOURCES:
568 @@ -383,67 +372,6 @@ class CLI(object):
569 for block in f:
570 self._parse_block(block)
571
572 - # Process all parsed networks from every RIR we happen to have access to,
573 - # insert the largest network chunks into the networks table immediately...
574 - families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
575 -
576 - for family in (row.family for row in families):
577 - smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
578 -
579 - self.db.execute("INSERT INTO networks(network, country) \
580 - SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
581 -
582 - # ... determine any other prefixes for this network family, ...
583 - prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
584 - WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
585 -
586 - # ... and insert networks with this prefix in case they provide additional
587 - # information (i. e. subnet of a larger chunk with a different country)
588 - for prefix in (row.prefix for row in prefixes):
589 - self.db.execute("""
590 - WITH candidates AS (
591 - SELECT
592 - _rirdata.network,
593 - _rirdata.country
594 - FROM
595 - _rirdata
596 - WHERE
597 - family(_rirdata.network) = %s
598 - AND
599 - masklen(_rirdata.network) = %s
600 - ),
601 - filtered AS (
602 - SELECT
603 - DISTINCT ON (c.network)
604 - c.network,
605 - c.country,
606 - masklen(networks.network),
607 - networks.country AS parent_country
608 - FROM
609 - candidates c
610 - LEFT JOIN
611 - networks
612 - ON
613 - c.network << networks.network
614 - ORDER BY
615 - c.network,
616 - masklen(networks.network) DESC NULLS LAST
617 - )
618 - INSERT INTO
619 - networks(network, country)
620 - SELECT
621 - network,
622 - country
623 - FROM
624 - filtered
625 - WHERE
626 - parent_country IS NULL
627 - OR
628 - country <> parent_country
629 - ON CONFLICT DO NOTHING""",
630 - family, prefix,
631 - )
632 -
633 self.db.execute("""
634 INSERT INTO autnums(number, name)
635 SELECT _autnums.number, _organizations.name FROM _autnums
636 @@ -544,30 +472,17 @@ class CLI(object):
637 inetnum[key] = val.upper()
638
639 # Skip empty objects
640 - if not inetnum or not "country" in inetnum:
641 + if not inetnum:
642 return
643
644 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
645
646 - # Bail out in case we have processed a network covering the entire IP range, which
647 - # is necessary to work around faulty (?) IPv6 network processing
648 - if network.prefixlen == 0:
649 - logging.warning("Skipping network covering the entire IP adress range: %s" % network)
650 - return
651 -
652 - # Bail out in case we have processed a network whose prefix length indicates it is
653 - # not globally routable (we have decided not to process them at the moment, as they
654 - # significantly enlarge our database without providing very helpful additional information)
655 - if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
656 - logging.info("Skipping network too small to be publicly announced: %s" % network)
657 - return
658 -
659 # Bail out in case we have processed a non-public IP network
660 if network.is_private:
661 logging.warning("Skipping non-globally routable network: %s" % network)
662 return
663
664 - self.db.execute("INSERT INTO _rirdata(network, country) \
665 + self.db.execute("INSERT INTO networks(network, country) \
666 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
667 "%s" % network, inetnum.get("country"),
668 )
669 --
670 2.20.1
671
672 From 44341478233115b26bb27fdb24da5b0a1eedb173 Mon Sep 17 00:00:00 2001
673 From: Michael Tremer <michael.tremer@ipfire.org>
674 Date: Fri, 16 Oct 2020 12:26:43 +0000
675 Subject: [PATCH 07/70] Revert "Revert "Revert "importer: Import raw sources
676 for inetnum's again"""
677
678 This reverts commit f532841e9197ce2f40aad8c086d786b2cb783a54.
679
680 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
681 ---
682 src/python/importer.py | 14 ++++----
683 src/python/location-importer.in | 63 ---------------------------------
684 2 files changed, 7 insertions(+), 70 deletions(-)
685
686 diff --git a/src/python/importer.py b/src/python/importer.py
687 index f19db4b..de20f37 100644
688 --- a/src/python/importer.py
689 +++ b/src/python/importer.py
690 @@ -30,8 +30,8 @@ WHOIS_SOURCES = (
691 "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
692
693 # Asia Pacific Network Information Centre
694 - "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
695 - "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
696 + #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
697 + #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
698 #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
699 #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
700 "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
701 @@ -45,8 +45,8 @@ WHOIS_SOURCES = (
702 # XXX ???
703
704 # Réseaux IP Européens
705 - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
706 - "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
707 + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
708 + #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
709 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
710 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
711 "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
712 @@ -55,10 +55,10 @@ WHOIS_SOURCES = (
713
714 EXTENDED_SOURCES = (
715 # African Network Information Centre
716 - #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
717 + "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
718
719 # Asia Pacific Network Information Centre
720 - #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
721 + "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
722
723 # American Registry for Internet Numbers
724 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
725 @@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
726 "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
727
728 # Réseaux IP Européens
729 - #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
730 + "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
731 )
732
733 class Downloader(object):
734 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
735 index b220eaf..e87d378 100644
736 --- a/src/python/location-importer.in
737 +++ b/src/python/location-importer.in
738 @@ -395,10 +395,6 @@ class CLI(object):
739 if line.startswith("aut-num:"):
740 return self._parse_autnum_block(block)
741
742 - # inetnum
743 - if line.startswith("inet6num:") or line.startswith("inetnum:"):
744 - return self._parse_inetnum_block(block)
745 -
746 # organisation
747 elif line.startswith("organisation:"):
748 return self._parse_org_block(block)
749 @@ -428,65 +424,6 @@ class CLI(object):
750 autnum.get("asn"), autnum.get("org"),
751 )
752
753 - def _parse_inetnum_block(self, block):
754 - logging.debug("Parsing inetnum block:")
755 -
756 - inetnum = {}
757 - for line in block:
758 - logging.debug(line)
759 -
760 - # Split line
761 - key, val = split_line(line)
762 -
763 - if key == "inetnum":
764 - start_address, delim, end_address = val.partition("-")
765 -
766 - # Strip any excess space
767 - start_address, end_address = start_address.rstrip(), end_address.strip()
768 -
769 - # Convert to IP address
770 - try:
771 - start_address = ipaddress.ip_address(start_address)
772 - end_address = ipaddress.ip_address(end_address)
773 - except ValueError:
774 - logging.warning("Could not parse line: %s" % line)
775 - return
776 -
777 - # Set prefix to default
778 - prefix = 32
779 -
780 - # Count number of addresses in this subnet
781 - num_addresses = int(end_address) - int(start_address)
782 - if num_addresses:
783 - prefix -= math.log(num_addresses, 2)
784 -
785 - inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
786 -
787 - elif key == "inet6num":
788 - inetnum[key] = val
789 -
790 - elif key == "country":
791 - if val == "UNITED STATES":
792 - val = "US"
793 -
794 - inetnum[key] = val.upper()
795 -
796 - # Skip empty objects
797 - if not inetnum:
798 - return
799 -
800 - network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
801 -
802 - # Bail out in case we have processed a non-public IP network
803 - if network.is_private:
804 - logging.warning("Skipping non-globally routable network: %s" % network)
805 - return
806 -
807 - self.db.execute("INSERT INTO networks(network, country) \
808 - VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
809 - "%s" % network, inetnum.get("country"),
810 - )
811 -
812 def _parse_org_block(self, block):
813 org = {}
814 for line in block:
815 --
816 2.20.1
817
818 From a7d3a7a0565a0e09d3442e5829a0f30f016993b9 Mon Sep 17 00:00:00 2001
819 From: Michael Tremer <michael.tremer@ipfire.org>
820 Date: Tue, 20 Oct 2020 20:44:43 +0000
821 Subject: [PATCH 08/70] as: Fix dereferencing NULL pointer when setting AS name
822
823 Reported-by: Gisle Vanem <gisle.vanem@gmail.com>
824 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
825 ---
826 src/as.c | 8 +++++++-
827 1 file changed, 7 insertions(+), 1 deletion(-)
828
829 diff --git a/src/as.c b/src/as.c
830 index e1fbb01..8421ac8 100644
831 --- a/src/as.c
832 +++ b/src/as.c
833 @@ -90,7 +90,13 @@ LOC_EXPORT const char* loc_as_get_name(struct loc_as* as) {
834 }
835
836 LOC_EXPORT int loc_as_set_name(struct loc_as* as, const char* name) {
837 - as->name = strdup(name);
838 + if (as->name)
839 + free(as->name);
840 +
841 + if (name)
842 + as->name = strdup(name);
843 + else
844 + as->name = NULL;
845
846 return 0;
847 }
848 --
849 2.20.1
850
851 From ddb326ad38a7c7202315dd2c6f938313db04ee22 Mon Sep 17 00:00:00 2001
852 From: Michael Tremer <michael.tremer@ipfire.org>
853 Date: Wed, 21 Oct 2020 09:18:08 +0000
854 Subject: [PATCH 09/70] as: Do not attempt to match name when it wasn't set
855
856 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
857 ---
858 src/as.c | 4 ++++
859 1 file changed, 4 insertions(+)
860
861 diff --git a/src/as.c b/src/as.c
862 index 8421ac8..757bf3d 100644
863 --- a/src/as.c
864 +++ b/src/as.c
865 @@ -145,6 +145,10 @@ int loc_as_match_string(struct loc_as* as, const char* string) {
866 if (!string)
867 return 1;
868
869 + // Cannot match anything when name is not set
870 + if (!as->name)
871 + return 1;
872 +
873 // Search if string is in name
874 if (strcasestr(as->name, string) != NULL)
875 return 1;
876 --
877 2.20.1
878
879 From d226ad2d97cbcd42ce807d9308569b1b9c5d4e2f Mon Sep 17 00:00:00 2001
880 From: Michael Tremer <michael.tremer@ipfire.org>
881 Date: Wed, 21 Oct 2020 09:28:39 +0000
882 Subject: [PATCH 10/70] writer: Free array with pointer to ASes, too
883
884 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
885 ---
886 src/writer.c | 7 +++++--
887 1 file changed, 5 insertions(+), 2 deletions(-)
888
889 diff --git a/src/writer.c b/src/writer.c
890 index 5939cff..160650f 100644
891 --- a/src/writer.c
892 +++ b/src/writer.c
893 @@ -147,8 +147,11 @@ static void loc_writer_free(struct loc_writer* writer) {
894 EVP_PKEY_free(writer->private_key2);
895
896 // Unref all AS
897 - for (unsigned int i = 0; i < writer->as_count; i++) {
898 - loc_as_unref(writer->as[i]);
899 + if (writer->as) {
900 + for (unsigned int i = 0; i < writer->as_count; i++) {
901 + loc_as_unref(writer->as[i]);
902 + }
903 + free(writer->as);
904 }
905
906 // Release network tree
907 --
908 2.20.1
909
910 From d89a7d62772048ae1bd18d03f69df46b7e1a3d3c Mon Sep 17 00:00:00 2001
911 From: Michael Tremer <michael.tremer@ipfire.org>
912 Date: Wed, 21 Oct 2020 09:31:29 +0000
913 Subject: [PATCH 11/70] writer: Free countries when the writer is being
914 destroyed
915
916 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
917 ---
918 src/writer.c | 8 ++++++++
919 1 file changed, 8 insertions(+)
920
921 diff --git a/src/writer.c b/src/writer.c
922 index 160650f..2f09b56 100644
923 --- a/src/writer.c
924 +++ b/src/writer.c
925 @@ -154,6 +154,14 @@ static void loc_writer_free(struct loc_writer* writer) {
926 free(writer->as);
927 }
928
929 + // Unref all countries
930 + if (writer->countries) {
931 + for (unsigned int i = 0; i < writer->countries_count; i++) {
932 + loc_country_unref(writer->countries[i]);
933 + }
934 + free(writer->countries);
935 + }
936 +
937 // Release network tree
938 if (writer->networks)
939 loc_network_tree_unref(writer->networks);
940 --
941 2.20.1
942
943 From 0f1aedbc68e3945770c93e0ebd83eed0f555d6f0 Mon Sep 17 00:00:00 2001
944 From: Michael Tremer <michael.tremer@ipfire.org>
945 Date: Wed, 21 Oct 2020 13:19:44 +0000
946 Subject: [PATCH 12/70] tests: Try adding an invalid network
947
948 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
949 ---
950 src/test-network.c | 8 ++++++++
951 1 file changed, 8 insertions(+)
952
953 diff --git a/src/test-network.c b/src/test-network.c
954 index d38f13d..e908b57 100644
955 --- a/src/test-network.c
956 +++ b/src/test-network.c
957 @@ -160,6 +160,14 @@ int main(int argc, char** argv) {
958 // Set ASN
959 loc_network_set_asn(network4, 1024);
960
961 + // Try adding an invalid network
962 + struct loc_network* network;
963 + err = loc_writer_add_network(writer, &network, "xxxx:xxxx::/32");
964 + if (err != -EINVAL) {
965 + fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
966 + exit(EXIT_FAILURE);
967 + }
968 +
969 FILE* f = tmpfile();
970 if (!f) {
971 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
972 --
973 2.20.1
974
975 From 13ad6e695f9ffc7847b3afe3e9cbcea8fb3a443f Mon Sep 17 00:00:00 2001
976 From: Michael Tremer <michael.tremer@ipfire.org>
977 Date: Wed, 21 Oct 2020 13:36:35 +0000
978 Subject: [PATCH 13/70] networks: Improve parsing IP addresses
979
980 loc_network_new_from_string() seem to have had some unexpected
981 behaviour for invalid inputs.
982
983 The function has been tidied up slightly and returns as soon as
984 some invalid input was detected.
985
986 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
987 ---
988 src/network.c | 52 +++++++++++++++++++++++++++++++---------------
989 src/test-network.c | 7 +++++++
990 2 files changed, 42 insertions(+), 17 deletions(-)
991
992 diff --git a/src/network.c b/src/network.c
993 index 366caa2..c112a41 100644
994 --- a/src/network.c
995 +++ b/src/network.c
996 @@ -160,9 +160,10 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network
997 LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_network** network,
998 const char* address_string) {
999 struct in6_addr first_address;
1000 - unsigned int prefix = 0;
1001 char* prefix_string;
1002 - int r = 1;
1003 + int r = -EINVAL;
1004 +
1005 + DEBUG(ctx, "Attempting to parse network %s\n", address_string);
1006
1007 // Make a copy of the string to work on it
1008 char* buffer = strdup(address_string);
1009 @@ -171,29 +172,46 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1010 // Split address and prefix
1011 address_string = strsep(&prefix_string, "/");
1012
1013 - // Did we find a prefix?
1014 - if (prefix_string) {
1015 - // Convert prefix to integer
1016 - prefix = strtol(prefix_string, NULL, 10);
1017 + DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
1018
1019 - if (prefix) {
1020 - // Parse the address
1021 - r = loc_parse_address(ctx, address_string, &first_address);
1022 + // We need to have a prefix
1023 + if (!prefix_string) {
1024 + DEBUG(ctx, "No prefix set\n");
1025 + goto FAIL;
1026 + }
1027
1028 - // Map the prefix to IPv6 if needed
1029 - if (IN6_IS_ADDR_V4MAPPED(&first_address))
1030 - prefix += 96;
1031 - }
1032 + // Convert prefix to integer
1033 + unsigned int prefix = strtol(prefix_string, NULL, 10);
1034 +
1035 + // End if the prefix was invalid
1036 + if (!prefix) {
1037 + DEBUG(ctx, "The prefix is zero or not a number\n");
1038 + goto FAIL;
1039 + }
1040 +
1041 + // Parse the address
1042 + r = loc_parse_address(ctx, address_string, &first_address);
1043 + if (r) {
1044 + DEBUG(ctx, "The address could not be parsed\n");
1045 + goto FAIL;
1046 }
1047
1048 + // Map the prefix to IPv6 if needed
1049 + if (IN6_IS_ADDR_V4MAPPED(&first_address))
1050 + prefix += 96;
1051 +
1052 +FAIL:
1053 // Free temporary buffer
1054 free(buffer);
1055
1056 - if (r == 0) {
1057 - r = loc_network_new(ctx, network, &first_address, prefix);
1058 - }
1059 + // Exit if the parsing was unsuccessful
1060 + if (r)
1061 + return r;
1062 +
1063 + DEBUG(ctx, "GOT HERE\n");
1064
1065 - return r;
1066 + // Create a new network
1067 + return loc_network_new(ctx, network, &first_address, prefix);
1068 }
1069
1070 LOC_EXPORT struct loc_network* loc_network_ref(struct loc_network* network) {
1071 diff --git a/src/test-network.c b/src/test-network.c
1072 index e908b57..85eca00 100644
1073 --- a/src/test-network.c
1074 +++ b/src/test-network.c
1075 @@ -168,6 +168,13 @@ int main(int argc, char** argv) {
1076 exit(EXIT_FAILURE);
1077 }
1078
1079 + // Try adding a single address
1080 + err = loc_writer_add_network(writer, &network, "2001:db8::");
1081 + if (err != -EINVAL) {
1082 + fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
1083 + exit(EXIT_FAILURE);
1084 + }
1085 +
1086 FILE* f = tmpfile();
1087 if (!f) {
1088 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
1089 --
1090 2.20.1
1091
1092 From 6a467e9345bb5a3d37911c9aaac30019eaa4492b Mon Sep 17 00:00:00 2001
1093 From: Michael Tremer <michael.tremer@ipfire.org>
1094 Date: Wed, 21 Oct 2020 13:43:21 +0000
1095 Subject: [PATCH 14/70] networks: Test if we can add localhost (IPv6)
1096
1097 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1098 ---
1099 src/test-network.c | 7 +++++++
1100 1 file changed, 7 insertions(+)
1101
1102 diff --git a/src/test-network.c b/src/test-network.c
1103 index 85eca00..8c7e898 100644
1104 --- a/src/test-network.c
1105 +++ b/src/test-network.c
1106 @@ -175,6 +175,13 @@ int main(int argc, char** argv) {
1107 exit(EXIT_FAILURE);
1108 }
1109
1110 + // Try adding localhost
1111 + err = loc_writer_add_network(writer, &network, "::1/128");
1112 + if (err != -EINVAL) {
1113 + fprintf(stderr, "It was possible to add localhost (::1/128): %d\n", err);
1114 + exit(EXIT_FAILURE);
1115 + }
1116 +
1117 FILE* f = tmpfile();
1118 if (!f) {
1119 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
1120 --
1121 2.20.1
1122
1123 From fc1190aa11e3ff3d2dbf5f4d408c298e7916f46f Mon Sep 17 00:00:00 2001
1124 From: Michael Tremer <michael.tremer@ipfire.org>
1125 Date: Wed, 21 Oct 2020 13:44:50 +0000
1126 Subject: [PATCH 15/70] networks: Remove accidentially committed debug line
1127
1128 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1129 ---
1130 src/network.c | 2 --
1131 1 file changed, 2 deletions(-)
1132
1133 diff --git a/src/network.c b/src/network.c
1134 index c112a41..be88d75 100644
1135 --- a/src/network.c
1136 +++ b/src/network.c
1137 @@ -208,8 +208,6 @@ FAIL:
1138 if (r)
1139 return r;
1140
1141 - DEBUG(ctx, "GOT HERE\n");
1142 -
1143 // Create a new network
1144 return loc_network_new(ctx, network, &first_address, prefix);
1145 }
1146 --
1147 2.20.1
1148
1149 From a1707d8983898b6878cdd5c68744bcc444e278ed Mon Sep 17 00:00:00 2001
1150 From: Michael Tremer <michael.tremer@ipfire.org>
1151 Date: Wed, 21 Oct 2020 13:53:36 +0000
1152 Subject: [PATCH 16/70] importer: Add search index to announcements table
1153
1154 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1155 ---
1156 src/python/location-importer.in | 1 +
1157 1 file changed, 1 insertion(+)
1158
1159 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1160 index e87d378..d0fe5a6 100644
1161 --- a/src/python/location-importer.in
1162 +++ b/src/python/location-importer.in
1163 @@ -152,6 +152,7 @@ class CLI(object):
1164 last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
1165 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
1166 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
1167 + CREATE INDEX IF NOT EXISTS announcements_search ON announcements USING GIST(network inet_ops);
1168
1169 -- autnums
1170 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
1171 --
1172 2.20.1
1173
1174 From 991baf530d47adb2ed7a15b65dc4565d07fa6d07 Mon Sep 17 00:00:00 2001
1175 From: Michael Tremer <michael.tremer@ipfire.org>
1176 Date: Wed, 21 Oct 2020 13:54:45 +0000
1177 Subject: [PATCH 17/70] importer: Add search index to network_overrides table
1178
1179 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1180 ---
1181 src/python/location-importer.in | 2 ++
1182 1 file changed, 2 insertions(+)
1183
1184 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1185 index d0fe5a6..fe21d73 100644
1186 --- a/src/python/location-importer.in
1187 +++ b/src/python/location-importer.in
1188 @@ -189,6 +189,8 @@ class CLI(object):
1189 );
1190 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
1191 ON network_overrides(network);
1192 + CREATE INDEX IF NOT EXISTS network_overrides_search
1193 + ON network_overrides USING GIST(network inet_ops);
1194 """)
1195
1196 return db
1197 --
1198 2.20.1
1199
1200 From bbea93a74651df10e2ffdbd09eb434dc6a0471bc Mon Sep 17 00:00:00 2001
1201 From: Michael Tremer <michael.tremer@ipfire.org>
1202 Date: Wed, 21 Oct 2020 16:01:57 +0000
1203 Subject: [PATCH 18/70] importer: Restructure SQL query to be executed in
1204 parallel
1205
1206 There are no functional changes, this just runs quicker now.
1207
1208 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1209 ---
1210 src/python/location-importer.in | 67 ++++++++++++++++++---------------
1211 1 file changed, 37 insertions(+), 30 deletions(-)
1212
1213 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1214 index fe21d73..3c1e5e2 100644
1215 --- a/src/python/location-importer.in
1216 +++ b/src/python/location-importer.in
1217 @@ -237,34 +237,24 @@ class CLI(object):
1218
1219 # Select all known networks
1220 rows = self.db.query("""
1221 - -- Get a (sorted) list of all known networks
1222 - WITH known_networks AS (
1223 - SELECT network FROM announcements
1224 - UNION
1225 - SELECT network FROM networks
1226 - UNION
1227 - SELECT network FROM network_overrides
1228 - ORDER BY network
1229 - )
1230 -
1231 -- Return a list of those networks enriched with all
1232 -- other information that we store in the database
1233 SELECT
1234 - DISTINCT ON (known_networks.network)
1235 - known_networks.network AS network,
1236 - announcements.autnum AS autnum,
1237 + DISTINCT ON (network)
1238 + network,
1239 + autnum,
1240
1241 -- Country
1242 COALESCE(
1243 (
1244 SELECT country FROM network_overrides overrides
1245 - WHERE announcements.network <<= overrides.network
1246 + WHERE networks.network <<= overrides.network
1247 ORDER BY masklen(overrides.network) DESC
1248 LIMIT 1
1249 ),
1250 (
1251 SELECT country FROM autnum_overrides overrides
1252 - WHERE announcements.autnum = overrides.number
1253 + WHERE networks.autnum = overrides.number
1254 ),
1255 networks.country
1256 ) AS country,
1257 @@ -273,50 +263,67 @@ class CLI(object):
1258 COALESCE(
1259 (
1260 SELECT is_anonymous_proxy FROM network_overrides overrides
1261 - WHERE announcements.network <<= overrides.network
1262 + WHERE networks.network <<= overrides.network
1263 ORDER BY masklen(overrides.network) DESC
1264 LIMIT 1
1265 ),
1266 (
1267 SELECT is_anonymous_proxy FROM autnum_overrides overrides
1268 - WHERE announcements.autnum = overrides.number
1269 + WHERE networks.autnum = overrides.number
1270 ),
1271 FALSE
1272 ) AS is_anonymous_proxy,
1273 COALESCE(
1274 (
1275 SELECT is_satellite_provider FROM network_overrides overrides
1276 - WHERE announcements.network <<= overrides.network
1277 + WHERE networks.network <<= overrides.network
1278 ORDER BY masklen(overrides.network) DESC
1279 LIMIT 1
1280 ),
1281 (
1282 SELECT is_satellite_provider FROM autnum_overrides overrides
1283 - WHERE announcements.autnum = overrides.number
1284 + WHERE networks.autnum = overrides.number
1285 ),
1286 FALSE
1287 ) AS is_satellite_provider,
1288 COALESCE(
1289 (
1290 SELECT is_anycast FROM network_overrides overrides
1291 - WHERE announcements.network <<= overrides.network
1292 + WHERE networks.network <<= overrides.network
1293 ORDER BY masklen(overrides.network) DESC
1294 LIMIT 1
1295 ),
1296 (
1297 SELECT is_anycast FROM autnum_overrides overrides
1298 - WHERE announcements.autnum = overrides.number
1299 + WHERE networks.autnum = overrides.number
1300 ),
1301 FALSE
1302 - ) AS is_anycast,
1303 -
1304 - -- Must be part of returned values for ORDER BY clause
1305 - masklen(announcements.network) AS sort_a,
1306 - masklen(networks.network) AS sort_b
1307 - FROM known_networks
1308 - LEFT JOIN announcements ON known_networks.network <<= announcements.network
1309 - LEFT JOIN networks ON known_networks.network <<= networks.network
1310 - ORDER BY known_networks.network, sort_a DESC, sort_b DESC
1311 + ) AS is_anycast
1312 + FROM (
1313 + SELECT
1314 + known_networks.network AS network,
1315 + announcements.autnum AS autnum,
1316 + networks.country AS country,
1317 +
1318 + -- Must be part of returned values for ORDER BY clause
1319 + masklen(announcements.network) AS sort_a,
1320 + masklen(networks.network) AS sort_b
1321 + FROM (
1322 + SELECT network FROM announcements
1323 + UNION ALL
1324 + SELECT network FROM networks
1325 + UNION ALL
1326 + SELECT network FROM network_overrides
1327 + ) known_networks
1328 + LEFT JOIN
1329 + announcements ON known_networks.network <<= announcements.network
1330 + LEFT JOIN
1331 + networks ON known_networks.network <<= networks.network
1332 + ORDER BY
1333 + known_networks.network,
1334 + sort_a DESC,
1335 + sort_b DESC
1336 + ) networks
1337 """)
1338
1339 for row in rows:
1340 --
1341 2.20.1
1342
1343 From 26ab419b68d166f932db1f97c38cb9d793d04187 Mon Sep 17 00:00:00 2001
1344 From: Michael Tremer <michael.tremer@ipfire.org>
1345 Date: Thu, 22 Oct 2020 12:24:34 +0000
1346 Subject: [PATCH 19/70] network: Allow adding single IP addresses and
1347 automatically add the prefix
1348
1349 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1350 ---
1351 src/network.c | 33 +++++++++++++++------------------
1352 src/test-network.c | 4 ++--
1353 2 files changed, 17 insertions(+), 20 deletions(-)
1354
1355 diff --git a/src/network.c b/src/network.c
1356 index be88d75..d7b1645 100644
1357 --- a/src/network.c
1358 +++ b/src/network.c
1359 @@ -161,6 +161,7 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1360 const char* address_string) {
1361 struct in6_addr first_address;
1362 char* prefix_string;
1363 + unsigned int prefix = 128;
1364 int r = -EINVAL;
1365
1366 DEBUG(ctx, "Attempting to parse network %s\n", address_string);
1367 @@ -174,21 +175,6 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1368
1369 DEBUG(ctx, " Split into address = %s, prefix = %s\n", address_string, prefix_string);
1370
1371 - // We need to have a prefix
1372 - if (!prefix_string) {
1373 - DEBUG(ctx, "No prefix set\n");
1374 - goto FAIL;
1375 - }
1376 -
1377 - // Convert prefix to integer
1378 - unsigned int prefix = strtol(prefix_string, NULL, 10);
1379 -
1380 - // End if the prefix was invalid
1381 - if (!prefix) {
1382 - DEBUG(ctx, "The prefix is zero or not a number\n");
1383 - goto FAIL;
1384 - }
1385 -
1386 // Parse the address
1387 r = loc_parse_address(ctx, address_string, &first_address);
1388 if (r) {
1389 @@ -196,9 +182,20 @@ LOC_EXPORT int loc_network_new_from_string(struct loc_ctx* ctx, struct loc_netwo
1390 goto FAIL;
1391 }
1392
1393 - // Map the prefix to IPv6 if needed
1394 - if (IN6_IS_ADDR_V4MAPPED(&first_address))
1395 - prefix += 96;
1396 + // If a prefix was given, we will try to parse it
1397 + if (prefix_string) {
1398 + // Convert prefix to integer
1399 + prefix = strtol(prefix_string, NULL, 10);
1400 +
1401 + if (!prefix) {
1402 + DEBUG(ctx, "The prefix was not parsable: %s\n", prefix_string);
1403 + goto FAIL;
1404 + }
1405 +
1406 + // Map the prefix to IPv6 if needed
1407 + if (IN6_IS_ADDR_V4MAPPED(&first_address))
1408 + prefix += 96;
1409 + }
1410
1411 FAIL:
1412 // Free temporary buffer
1413 diff --git a/src/test-network.c b/src/test-network.c
1414 index 8c7e898..b6776b4 100644
1415 --- a/src/test-network.c
1416 +++ b/src/test-network.c
1417 @@ -170,8 +170,8 @@ int main(int argc, char** argv) {
1418
1419 // Try adding a single address
1420 err = loc_writer_add_network(writer, &network, "2001:db8::");
1421 - if (err != -EINVAL) {
1422 - fprintf(stderr, "It was possible to add an invalid network (err = %d)\n", err);
1423 + if (err) {
1424 + fprintf(stderr, "It was impossible to add an single IP address (err = %d)\n", err);
1425 exit(EXIT_FAILURE);
1426 }
1427
1428 --
1429 2.20.1
1430
1431 From aadac4c569e921be1d28dd3b2377ac7f3732213e Mon Sep 17 00:00:00 2001
1432 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
1433 Date: Wed, 21 Oct 2020 14:47:36 +0000
1434 Subject: [PATCH 20/70] Revert "Revert "Revert "Revert "importer: Import raw
1435 sources for inetnum's again""""
1436 MIME-Version: 1.0
1437 Content-Type: text/plain; charset=UTF-8
1438 Content-Transfer-Encoding: 8bit
1439
1440 This reverts commit 44341478233115b26bb27fdb24da5b0a1eedb173.
1441
1442 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
1443 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1444 ---
1445 src/python/importer.py | 14 ++++----
1446 src/python/location-importer.in | 63 +++++++++++++++++++++++++++++++++
1447 2 files changed, 70 insertions(+), 7 deletions(-)
1448
1449 diff --git a/src/python/importer.py b/src/python/importer.py
1450 index de20f37..f19db4b 100644
1451 --- a/src/python/importer.py
1452 +++ b/src/python/importer.py
1453 @@ -30,8 +30,8 @@ WHOIS_SOURCES = (
1454 "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
1455
1456 # Asia Pacific Network Information Centre
1457 - #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
1458 - #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
1459 + "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
1460 + "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
1461 #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
1462 #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
1463 "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
1464 @@ -45,8 +45,8 @@ WHOIS_SOURCES = (
1465 # XXX ???
1466
1467 # Réseaux IP Européens
1468 - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
1469 - #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
1470 + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
1471 + "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
1472 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
1473 #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
1474 "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
1475 @@ -55,10 +55,10 @@ WHOIS_SOURCES = (
1476
1477 EXTENDED_SOURCES = (
1478 # African Network Information Centre
1479 - "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
1480 + #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
1481
1482 # Asia Pacific Network Information Centre
1483 - "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
1484 + #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
1485
1486 # American Registry for Internet Numbers
1487 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
1488 @@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
1489 "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
1490
1491 # Réseaux IP Européens
1492 - "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
1493 + #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
1494 )
1495
1496 class Downloader(object):
1497 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1498 index 3c1e5e2..e8a4fc5 100644
1499 --- a/src/python/location-importer.in
1500 +++ b/src/python/location-importer.in
1501 @@ -405,6 +405,10 @@ class CLI(object):
1502 if line.startswith("aut-num:"):
1503 return self._parse_autnum_block(block)
1504
1505 + # inetnum
1506 + if line.startswith("inet6num:") or line.startswith("inetnum:"):
1507 + return self._parse_inetnum_block(block)
1508 +
1509 # organisation
1510 elif line.startswith("organisation:"):
1511 return self._parse_org_block(block)
1512 @@ -434,6 +438,65 @@ class CLI(object):
1513 autnum.get("asn"), autnum.get("org"),
1514 )
1515
1516 + def _parse_inetnum_block(self, block):
1517 + logging.debug("Parsing inetnum block:")
1518 +
1519 + inetnum = {}
1520 + for line in block:
1521 + logging.debug(line)
1522 +
1523 + # Split line
1524 + key, val = split_line(line)
1525 +
1526 + if key == "inetnum":
1527 + start_address, delim, end_address = val.partition("-")
1528 +
1529 + # Strip any excess space
1530 + start_address, end_address = start_address.rstrip(), end_address.strip()
1531 +
1532 + # Convert to IP address
1533 + try:
1534 + start_address = ipaddress.ip_address(start_address)
1535 + end_address = ipaddress.ip_address(end_address)
1536 + except ValueError:
1537 + logging.warning("Could not parse line: %s" % line)
1538 + return
1539 +
1540 + # Set prefix to default
1541 + prefix = 32
1542 +
1543 + # Count number of addresses in this subnet
1544 + num_addresses = int(end_address) - int(start_address)
1545 + if num_addresses:
1546 + prefix -= math.log(num_addresses, 2)
1547 +
1548 + inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
1549 +
1550 + elif key == "inet6num":
1551 + inetnum[key] = val
1552 +
1553 + elif key == "country":
1554 + if val == "UNITED STATES":
1555 + val = "US"
1556 +
1557 + inetnum[key] = val.upper()
1558 +
1559 + # Skip empty objects
1560 + if not inetnum:
1561 + return
1562 +
1563 + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
1564 +
1565 + # Bail out in case we have processed a non-public IP network
1566 + if network.is_private:
1567 + logging.warning("Skipping non-globally routable network: %s" % network)
1568 + return
1569 +
1570 + self.db.execute("INSERT INTO networks(network, country) \
1571 + VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
1572 + "%s" % network, inetnum.get("country"),
1573 + )
1574 +
1575 def _parse_org_block(self, block):
1576 org = {}
1577 for line in block:
1578 --
1579 2.20.1
1580
1581 From 002deb6b42ac0b3624c07e3352cebd72dc0685a2 Mon Sep 17 00:00:00 2001
1582 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
1583 Date: Wed, 21 Oct 2020 14:47:37 +0000
1584 Subject: [PATCH 21/70] Revert "Revert "location-importer.in: only import
1585 relevant data from AFRINIC, APNIC and RIPE""
1586 MIME-Version: 1.0
1587 Content-Type: text/plain; charset=UTF-8
1588 Content-Transfer-Encoding: 8bit
1589
1590 This reverts commit 13f67f285856e8eabfeff2daf1be3aeaa36a82cc.
1591
1592 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
1593 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1594 ---
1595 src/python/location-importer.in | 89 ++++++++++++++++++++++++++++++++-
1596 1 file changed, 87 insertions(+), 2 deletions(-)
1597
1598 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1599 index e8a4fc5..5656c41 100644
1600 --- a/src/python/location-importer.in
1601 +++ b/src/python/location-importer.in
1602 @@ -166,6 +166,7 @@ class CLI(object):
1603 -- networks
1604 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
1605 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
1606 + CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
1607 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
1608
1609 -- overrides
1610 @@ -375,6 +376,16 @@ class CLI(object):
1611 CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
1612 ON COMMIT DROP;
1613 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
1614 +
1615 + CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
1616 + ON COMMIT DROP;
1617 + CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
1618 + CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
1619 + """)
1620 +
1621 + # Remove all previously imported content
1622 + self.db.execute("""
1623 + TRUNCATE TABLE networks;
1624 """)
1625
1626 for source in location.importer.WHOIS_SOURCES:
1627 @@ -382,6 +393,67 @@ class CLI(object):
1628 for block in f:
1629 self._parse_block(block)
1630
1631 + # Process all parsed networks from every RIR we happen to have access to,
1632 + # insert the largest network chunks into the networks table immediately...
1633 + families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
1634 +
1635 + for family in (row.family for row in families):
1636 + smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
1637 +
1638 + self.db.execute("INSERT INTO networks(network, country) \
1639 + SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
1640 +
1641 + # ... determine any other prefixes for this network family, ...
1642 + prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
1643 + WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
1644 +
1645 + # ... and insert networks with this prefix in case they provide additional
1646 + # information (i. e. subnet of a larger chunk with a different country)
1647 + for prefix in (row.prefix for row in prefixes):
1648 + self.db.execute("""
1649 + WITH candidates AS (
1650 + SELECT
1651 + _rirdata.network,
1652 + _rirdata.country
1653 + FROM
1654 + _rirdata
1655 + WHERE
1656 + family(_rirdata.network) = %s
1657 + AND
1658 + masklen(_rirdata.network) = %s
1659 + ),
1660 + filtered AS (
1661 + SELECT
1662 + DISTINCT ON (c.network)
1663 + c.network,
1664 + c.country,
1665 + masklen(networks.network),
1666 + networks.country AS parent_country
1667 + FROM
1668 + candidates c
1669 + LEFT JOIN
1670 + networks
1671 + ON
1672 + c.network << networks.network
1673 + ORDER BY
1674 + c.network,
1675 + masklen(networks.network) DESC NULLS LAST
1676 + )
1677 + INSERT INTO
1678 + networks(network, country)
1679 + SELECT
1680 + network,
1681 + country
1682 + FROM
1683 + filtered
1684 + WHERE
1685 + parent_country IS NULL
1686 + OR
1687 + country <> parent_country
1688 + ON CONFLICT DO NOTHING""",
1689 + family, prefix,
1690 + )
1691 +
1692 self.db.execute("""
1693 INSERT INTO autnums(number, name)
1694 SELECT _autnums.number, _organizations.name FROM _autnums
1695 @@ -482,17 +554,30 @@ class CLI(object):
1696 inetnum[key] = val.upper()
1697
1698 # Skip empty objects
1699 - if not inetnum:
1700 + if not inetnum or not "country" in inetnum:
1701 return
1702
1703 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
1704
1705 + # Bail out in case we have processed a network covering the entire IP range, which
1706 + # is necessary to work around faulty (?) IPv6 network processing
1707 + if network.prefixlen == 0:
1708 + logging.warning("Skipping network covering the entire IP adress range: %s" % network)
1709 + return
1710 +
1711 + # Bail out in case we have processed a network whose prefix length indicates it is
1712 + # not globally routable (we have decided not to process them at the moment, as they
1713 + # significantly enlarge our database without providing very helpful additional information)
1714 + if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
1715 + logging.info("Skipping network too small to be publicly announced: %s" % network)
1716 + return
1717 +
1718 # Bail out in case we have processed a non-public IP network
1719 if network.is_private:
1720 logging.warning("Skipping non-globally routable network: %s" % network)
1721 return
1722
1723 - self.db.execute("INSERT INTO networks(network, country) \
1724 + self.db.execute("INSERT INTO _rirdata(network, country) \
1725 VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
1726 "%s" % network, inetnum.get("country"),
1727 )
1728 --
1729 2.20.1
1730
1731 From 28c73fa3f4257e0a41e52af8a9643da414a6cb6f Mon Sep 17 00:00:00 2001
1732 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
1733 Date: Wed, 21 Oct 2020 14:47:38 +0000
1734 Subject: [PATCH 22/70] export.py: fix exporting IP networks for crappy
1735 xt_geoip module
1736 MIME-Version: 1.0
1737 Content-Type: text/plain; charset=UTF-8
1738 Content-Transfer-Encoding: 8bit
1739
1740 In contrast to the location database itself, the xt_geoip module
1741 consumes a list of IP networks for each country, and returns after the
1742 first match.
1743
1744 We therefore need to...
1745
1746 (a) sort IP networks by their size, allow as precise matches as possible
1747 (b) export _any_ IP networks - including inverted subnets - to prevent
1748 undefined overlaps
1749 (c) do the entire thing as fast as possible, consuming as less disk
1750 space as possible, which is why we can't just iterate over all /24
1751 chunks
1752
1753 Partially fixes: #12499
1754
1755 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1756 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
1757 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1758 ---
1759 src/python/export.py | 69 ++++++++++++++++++++++++++++++++++----------
1760 1 file changed, 54 insertions(+), 15 deletions(-)
1761
1762 diff --git a/src/python/export.py b/src/python/export.py
1763 index d15c6f0..5eaf43f 100644
1764 --- a/src/python/export.py
1765 +++ b/src/python/export.py
1766 @@ -39,8 +39,8 @@ class OutputWriter(object):
1767 suffix = "networks"
1768 mode = "w"
1769
1770 - def __init__(self, f, prefix=None, flatten=True):
1771 - self.f, self.prefix, self.flatten = f, prefix, flatten
1772 + def __init__(self, db, f, prefix=None, flatten=True):
1773 + self.db, self.f, self.prefix, self.flatten = db, f, prefix, flatten
1774
1775 # The previously written network
1776 self._last_network = None
1777 @@ -49,13 +49,13 @@ class OutputWriter(object):
1778 self._write_header()
1779
1780 @classmethod
1781 - def open(cls, filename, **kwargs):
1782 + def open(cls, db, filename, **kwargs):
1783 """
1784 Convenience function to open a file
1785 """
1786 f = open(filename, cls.mode)
1787
1788 - return cls(f, **kwargs)
1789 + return cls(db, f, **kwargs)
1790
1791 def __repr__(self):
1792 return "<%s f=%s>" % (self.__class__.__name__, self.f)
1793 @@ -87,13 +87,31 @@ class OutputWriter(object):
1794 def _write_network(self, network):
1795 self.f.write("%s\n" % network)
1796
1797 - def write(self, network):
1798 + def write(self, network, subnets):
1799 if self.flatten and self._flatten(network):
1800 log.debug("Skipping writing network %s" % network)
1801 return
1802
1803 - # Write the network to file
1804 - self._write_network(network)
1805 + # Write the network when it has no subnets
1806 + if not subnets:
1807 + network = ipaddress.ip_network("%s" % network)
1808 + return self._write_network(network)
1809 +
1810 + # Collect all matching subnets
1811 + matching_subnets = []
1812 +
1813 + for subnet in sorted(subnets):
1814 + # Try to find the subnet in the database
1815 + n = self.db.lookup("%s" % subnet.network_address)
1816 +
1817 + # No entry or matching country means those IP addresses belong here
1818 + if not n or n.country_code == network.country_code:
1819 + matching_subnets.append(subnet)
1820 +
1821 + # Write all networks as compact as possible
1822 + for network in ipaddress.collapse_addresses(matching_subnets):
1823 + log.debug("Writing %s to database" % network)
1824 + self._write_network(network)
1825
1826 def finish(self):
1827 """
1828 @@ -143,10 +161,10 @@ class XTGeoIPOutputWriter(OutputWriter):
1829 mode = "wb"
1830
1831 def _write_network(self, network):
1832 - for address in (network.first_address, network.last_address):
1833 + for address in (network.network_address, network.broadcast_address):
1834 # Convert this into a string of bits
1835 bytes = socket.inet_pton(
1836 - network.family, address,
1837 + socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
1838 )
1839
1840 self.f.write(bytes)
1841 @@ -175,7 +193,7 @@ class Exporter(object):
1842 directory, prefix=country_code, suffix=self.writer.suffix, family=family,
1843 )
1844
1845 - writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code)
1846 + writers[country_code] = self.writer.open(self.db, filename, prefix="CC_%s" % country_code)
1847
1848 # Create writers for ASNs
1849 for asn in asns:
1850 @@ -183,22 +201,43 @@ class Exporter(object):
1851 directory, "AS%s" % asn, suffix=self.writer.suffix, family=family,
1852 )
1853
1854 - writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
1855 + writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
1856
1857 # Get all networks that match the family
1858 networks = self.db.search_networks(family=family)
1859
1860 + # Materialise the generator into a list (uses quite some memory)
1861 + networks = list(networks)
1862 +
1863 # Walk through all networks
1864 - for network in networks:
1865 + for i, network in enumerate(networks):
1866 + _network = ipaddress.ip_network("%s" % network)
1867 +
1868 + # Search for all subnets
1869 + subnets = set()
1870 +
1871 + while i < len(networks):
1872 + subnet = networks[i+1]
1873 +
1874 + if subnet.is_subnet_of(network):
1875 + _subnet = ipaddress.ip_network("%s" % subnet)
1876 +
1877 + subnets.add(_subnet)
1878 + subnets.update(_network.address_exclude(_subnet))
1879 +
1880 + i += 1
1881 + else:
1882 + break
1883 +
1884 # Write matching countries
1885 try:
1886 - writers[network.country_code].write(network)
1887 + writers[network.country_code].write(network, subnets)
1888 except KeyError:
1889 pass
1890
1891 # Write matching ASNs
1892 try:
1893 - writers[network.asn].write(network)
1894 + writers[network.asn].write(network, subnets)
1895 except KeyError:
1896 pass
1897
1898 @@ -209,7 +248,7 @@ class Exporter(object):
1899 country = flags[flag]
1900
1901 try:
1902 - writers[country].write(network)
1903 + writers[country].write(network, subnets)
1904 except KeyError:
1905 pass
1906
1907 --
1908 2.20.1
1909
1910 From bd341642fc6bbcc050e9b4ec5124585c83cab84d Mon Sep 17 00:00:00 2001
1911 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
1912 Date: Wed, 21 Oct 2020 14:47:39 +0000
1913 Subject: [PATCH 23/70] location-importer.in: filter bogus IP networks for both
1914 Whois and extended sources
1915 MIME-Version: 1.0
1916 Content-Type: text/plain; charset=UTF-8
1917 Content-Transfer-Encoding: 8bit
1918
1919 Sanity checks for parsed networks have been put into a separate function
1920 to avoid boilerplate code for extended sources. This makes the location
1921 database less vulnerable to garbage written into RIR databases on
1922 purpose or by chance.
1923
1924 Fixes: #12500
1925
1926 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
1927 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
1928 ---
1929 src/python/location-importer.in | 83 ++++++++++++++++++++++++++-------
1930 1 file changed, 67 insertions(+), 16 deletions(-)
1931
1932 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
1933 index 5656c41..f24d357 100644
1934 --- a/src/python/location-importer.in
1935 +++ b/src/python/location-importer.in
1936 @@ -469,6 +469,69 @@ class CLI(object):
1937 for line in f:
1938 self._parse_line(line)
1939
1940 + def _check_parsed_network(self, network):
1941 + """
1942 + Assistive function to detect and subsequently sort out parsed
1943 + networks from RIR data (both Whois and so-called "extended sources"),
1944 + which are or have...
1945 +
1946 + (a) not globally routable (RFC 1918 space, et al.)
1947 + (b) covering a too large chunk of the IP address space (prefix length
1948 + is < 7 for IPv4 networks, and < 10 for IPv6)
1949 + (c) "0.0.0.0" or "::" as a network address
1950 + (d) are too small for being publicly announced (we have decided not to
1951 + process them at the moment, as they significantly enlarge our
1952 + database without providing very helpful additional information)
1953 +
1954 + This unfortunately is necessary due to brain-dead clutter across
1955 + various RIR databases, causing mismatches and eventually disruptions.
1956 +
1957 + We will return False in case a network is not suitable for adding
1958 + it to our database, and True otherwise.
1959 + """
1960 +
1961 + if not network or not (isinstance(network, ipaddress.IPv4Network) or isinstance(network, ipaddress.IPv6Network)):
1962 + return False
1963 +
1964 + if not network.is_global:
1965 + logging.warning("Skipping non-globally routable network: %s" % network)
1966 + return False
1967 +
1968 + if network.version == 4:
1969 + if network.prefixlen < 7:
1970 + logging.warning("Skipping too big IP chunk: %s" % network)
1971 + return False
1972 +
1973 + if network.prefixlen > 24:
1974 + logging.info("Skipping network too small to be publicly announced: %s" % network)
1975 + return False
1976 +
1977 + if str(network.network_address) == "0.0.0.0":
1978 + logging.warning("Skipping network based on 0.0.0.0: %s" % network)
1979 + return False
1980 +
1981 + elif network.version == 6:
1982 + if network.prefixlen < 10:
1983 + logging.warning("Skipping too big IP chunk: %s" % network)
1984 + return False
1985 +
1986 + if network.prefixlen > 48:
1987 + logging.info("Skipping network too small to be publicly announced: %s" % network)
1988 + return False
1989 +
1990 + if str(network.network_address) == "::":
1991 + logging.warning("Skipping network based on '::': %s" % network)
1992 + return False
1993 +
1994 + else:
1995 + # This should not happen...
1996 + logging.warning("Skipping network of unknown family, this should not happen: %s" % network)
1997 + return False
1998 +
1999 + # In case we have made it here, the network is considered to
2000 + # be suitable for libloc consumption...
2001 + return True
2002 +
2003 def _parse_block(self, block):
2004 # Get first line to find out what type of block this is
2005 line = block[0]
2006 @@ -559,22 +622,7 @@ class CLI(object):
2007
2008 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
2009
2010 - # Bail out in case we have processed a network covering the entire IP range, which
2011 - # is necessary to work around faulty (?) IPv6 network processing
2012 - if network.prefixlen == 0:
2013 - logging.warning("Skipping network covering the entire IP adress range: %s" % network)
2014 - return
2015 -
2016 - # Bail out in case we have processed a network whose prefix length indicates it is
2017 - # not globally routable (we have decided not to process them at the moment, as they
2018 - # significantly enlarge our database without providing very helpful additional information)
2019 - if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
2020 - logging.info("Skipping network too small to be publicly announced: %s" % network)
2021 - return
2022 -
2023 - # Bail out in case we have processed a non-public IP network
2024 - if network.is_private:
2025 - logging.warning("Skipping non-globally routable network: %s" % network)
2026 + if not self._check_parsed_network(network):
2027 return
2028
2029 self.db.execute("INSERT INTO _rirdata(network, country) \
2030 @@ -658,6 +706,9 @@ class CLI(object):
2031 log.warning("Invalid IP address: %s" % address)
2032 return
2033
2034 + if not self._check_parsed_network(network):
2035 + return
2036 +
2037 self.db.execute("INSERT INTO networks(network, country) \
2038 VALUES(%s, %s) ON CONFLICT (network) DO \
2039 UPDATE SET country = excluded.country",
2040 --
2041 2.20.1
2042
2043 From eee65490a10e0fe89b3834b8be176fc900084fa0 Mon Sep 17 00:00:00 2001
2044 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2045 Date: Wed, 21 Oct 2020 14:47:40 +0000
2046 Subject: [PATCH 24/70] importer.py: fetch LACNIC data via HTTPS
2047 MIME-Version: 1.0
2048 Content-Type: text/plain; charset=UTF-8
2049 Content-Transfer-Encoding: 8bit
2050
2051 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2052 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2053 ---
2054 src/python/importer.py | 2 +-
2055 1 file changed, 1 insertion(+), 1 deletion(-)
2056
2057 diff --git a/src/python/importer.py b/src/python/importer.py
2058 index f19db4b..5f46bc3 100644
2059 --- a/src/python/importer.py
2060 +++ b/src/python/importer.py
2061 @@ -64,7 +64,7 @@ EXTENDED_SOURCES = (
2062 "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
2063
2064 # Latin America and Caribbean Network Information Centre
2065 - "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
2066 + "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
2067
2068 # Réseaux IP Européens
2069 #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
2070 --
2071 2.20.1
2072
2073 From 84187ab5436eb158529d6f5e2a38890b4af3ddb4 Mon Sep 17 00:00:00 2001
2074 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2075 Date: Wed, 21 Oct 2020 14:47:41 +0000
2076 Subject: [PATCH 25/70] location-importer.in: omit historic/orphaned RIR data
2077 MIME-Version: 1.0
2078 Content-Type: text/plain; charset=UTF-8
2079 Content-Transfer-Encoding: 8bit
2080
2081 Some RIRs include detailled information regarding networks not managed
2082 by or allocated to themselves, particually APNIC. We need to filter
2083 those networks (they usually have a characteristic network name) in
2084 order to prevent operational quirks or returning wrong country codes.
2085
2086 Fixes: #12501
2087 Partially fixes: #12499
2088
2089 Cc: Michael Tremer <michael.tremer@ipfire.org>
2090 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2091 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2092 ---
2093 src/python/location-importer.in | 38 +++++++++++++++++++++------------
2094 1 file changed, 24 insertions(+), 14 deletions(-)
2095
2096 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
2097 index f24d357..a869256 100644
2098 --- a/src/python/location-importer.in
2099 +++ b/src/python/location-importer.in
2100 @@ -494,38 +494,38 @@ class CLI(object):
2101 return False
2102
2103 if not network.is_global:
2104 - logging.warning("Skipping non-globally routable network: %s" % network)
2105 + log.warning("Skipping non-globally routable network: %s" % network)
2106 return False
2107
2108 if network.version == 4:
2109 if network.prefixlen < 7:
2110 - logging.warning("Skipping too big IP chunk: %s" % network)
2111 + log.warning("Skipping too big IP chunk: %s" % network)
2112 return False
2113
2114 if network.prefixlen > 24:
2115 - logging.info("Skipping network too small to be publicly announced: %s" % network)
2116 + log.info("Skipping network too small to be publicly announced: %s" % network)
2117 return False
2118
2119 if str(network.network_address) == "0.0.0.0":
2120 - logging.warning("Skipping network based on 0.0.0.0: %s" % network)
2121 + log.warning("Skipping network based on 0.0.0.0: %s" % network)
2122 return False
2123
2124 elif network.version == 6:
2125 if network.prefixlen < 10:
2126 - logging.warning("Skipping too big IP chunk: %s" % network)
2127 + log.warning("Skipping too big IP chunk: %s" % network)
2128 return False
2129
2130 if network.prefixlen > 48:
2131 - logging.info("Skipping network too small to be publicly announced: %s" % network)
2132 + log.info("Skipping network too small to be publicly announced: %s" % network)
2133 return False
2134
2135 if str(network.network_address) == "::":
2136 - logging.warning("Skipping network based on '::': %s" % network)
2137 + log.warning("Skipping network based on '::': %s" % network)
2138 return False
2139
2140 else:
2141 # This should not happen...
2142 - logging.warning("Skipping network of unknown family, this should not happen: %s" % network)
2143 + log.warning("Skipping network of unknown family, this should not happen: %s" % network)
2144 return False
2145
2146 # In case we have made it here, the network is considered to
2147 @@ -574,15 +574,22 @@ class CLI(object):
2148 )
2149
2150 def _parse_inetnum_block(self, block):
2151 - logging.debug("Parsing inetnum block:")
2152 + log.debug("Parsing inetnum block:")
2153
2154 inetnum = {}
2155 for line in block:
2156 - logging.debug(line)
2157 + log.debug(line)
2158
2159 # Split line
2160 key, val = split_line(line)
2161
2162 + # Filter any inetnum records which are only referring to IP space
2163 + # not managed by that specific RIR...
2164 + if key == "netname":
2165 + if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()):
2166 + log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip())
2167 + return
2168 +
2169 if key == "inetnum":
2170 start_address, delim, end_address = val.partition("-")
2171
2172 @@ -594,7 +601,7 @@ class CLI(object):
2173 start_address = ipaddress.ip_address(start_address)
2174 end_address = ipaddress.ip_address(end_address)
2175 except ValueError:
2176 - logging.warning("Could not parse line: %s" % line)
2177 + log.warning("Could not parse line: %s" % line)
2178 return
2179
2180 # Set prefix to default
2181 @@ -611,15 +618,18 @@ class CLI(object):
2182 inetnum[key] = val
2183
2184 elif key == "country":
2185 - if val == "UNITED STATES":
2186 - val = "US"
2187 -
2188 inetnum[key] = val.upper()
2189
2190 # Skip empty objects
2191 if not inetnum or not "country" in inetnum:
2192 return
2193
2194 + # Skip objects with bogus country code 'ZZ'
2195 + if inetnum.get("country") == "ZZ":
2196 + log.warning("Skipping network with bogus country 'ZZ': %s" % \
2197 + (inetnum.get("inet6num") or inetnum.get("inetnum")))
2198 + return
2199 +
2200 network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
2201
2202 if not self._check_parsed_network(network):
2203 --
2204 2.20.1
2205
2206 From ebb087cfa30ec5ca0c96dcce66a91245c1ffc271 Mon Sep 17 00:00:00 2001
2207 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2208 Date: Wed, 21 Oct 2020 14:47:43 +0000
2209 Subject: [PATCH 26/70] location-importer.in: avoid log spam for too small
2210 networks
2211 MIME-Version: 1.0
2212 Content-Type: text/plain; charset=UTF-8
2213 Content-Transfer-Encoding: 8bit
2214
2215 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2216 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2217 ---
2218 src/python/location-importer.in | 4 ++--
2219 1 file changed, 2 insertions(+), 2 deletions(-)
2220
2221 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
2222 index a869256..864eab1 100644
2223 --- a/src/python/location-importer.in
2224 +++ b/src/python/location-importer.in
2225 @@ -503,7 +503,7 @@ class CLI(object):
2226 return False
2227
2228 if network.prefixlen > 24:
2229 - log.info("Skipping network too small to be publicly announced: %s" % network)
2230 + log.debug("Skipping network too small to be publicly announced: %s" % network)
2231 return False
2232
2233 if str(network.network_address) == "0.0.0.0":
2234 @@ -516,7 +516,7 @@ class CLI(object):
2235 return False
2236
2237 if network.prefixlen > 48:
2238 - log.info("Skipping network too small to be publicly announced: %s" % network)
2239 + log.debug("Skipping network too small to be publicly announced: %s" % network)
2240 return False
2241
2242 if str(network.network_address) == "::":
2243 --
2244 2.20.1
2245
2246 From bbed1fd2330e8efa6b413dc152a1a6ef2d771aac Mon Sep 17 00:00:00 2001
2247 From: Michael Tremer <michael.tremer@ipfire.org>
2248 Date: Tue, 27 Oct 2020 17:14:30 +0000
2249 Subject: [PATCH 27/70] export: Flatten the tree before exporting it
2250
2251 This patch removes the possibility that any IP address ranges
2252 might show up in multiple exported files.
2253
2254 If this was content from the database:
2255
2256 * 10.0.0.0/16 - DE
2257 * 10.0.1.0/24 - FR
2258
2259 Then the IP address 10.0.1.1 would match for both countries.
2260
2261 The algorithm will now break the larger /16 subnet apart into
2262 smaller subnets so that 10.0.1.0/24 is no longer overlapped.
2263
2264 There was some time spent on this to make this as efficient
2265 as possible.
2266
2267 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2268 ---
2269 src/python/export.py | 154 ++++++++++++++++++++++++++++++-------------
2270 1 file changed, 110 insertions(+), 44 deletions(-)
2271
2272 diff --git a/src/python/export.py b/src/python/export.py
2273 index 5eaf43f..dd44332 100644
2274 --- a/src/python/export.py
2275 +++ b/src/python/export.py
2276 @@ -89,28 +89,55 @@ class OutputWriter(object):
2277
2278 def write(self, network, subnets):
2279 if self.flatten and self._flatten(network):
2280 - log.debug("Skipping writing network %s" % network)
2281 + log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
2282 return
2283
2284 + # Convert network into a Python object
2285 + _network = ipaddress.ip_network("%s" % network)
2286 +
2287 # Write the network when it has no subnets
2288 if not subnets:
2289 - network = ipaddress.ip_network("%s" % network)
2290 - return self._write_network(network)
2291 + log.debug("Writing %s to %s" % (_network, self.f))
2292 + return self._write_network(_network)
2293 +
2294 + # Convert subnets into Python objects
2295 + _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets]
2296 +
2297 + # Split the network into smaller bits so that
2298 + # we can accomodate for any gaps in it later
2299 + to_check = set()
2300 + for _subnet in _subnets:
2301 + to_check.update(
2302 + _network.address_exclude(_subnet)
2303 + )
2304 +
2305 + # Clear the list of all subnets
2306 + subnets = []
2307 +
2308 + # Check if all subnets to not overlap with anything else
2309 + while to_check:
2310 + subnet_to_check = to_check.pop()
2311
2312 - # Collect all matching subnets
2313 - matching_subnets = []
2314 + for _subnet in _subnets:
2315 + # Drop this subnet if it equals one of the subnets
2316 + # or if it is subnet of one of them
2317 + if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet):
2318 + break
2319
2320 - for subnet in sorted(subnets):
2321 - # Try to find the subnet in the database
2322 - n = self.db.lookup("%s" % subnet.network_address)
2323 + # Break it down if it overlaps
2324 + if subnet_to_check.overlaps(_subnet):
2325 + to_check.update(
2326 + subnet_to_check.address_exclude(_subnet)
2327 + )
2328 + break
2329
2330 - # No entry or matching country means those IP addresses belong here
2331 - if not n or n.country_code == network.country_code:
2332 - matching_subnets.append(subnet)
2333 + # Add the subnet again as it passed the check
2334 + else:
2335 + subnets.append(subnet_to_check)
2336
2337 # Write all networks as compact as possible
2338 - for network in ipaddress.collapse_addresses(matching_subnets):
2339 - log.debug("Writing %s to database" % network)
2340 + for network in ipaddress.collapse_addresses(subnets):
2341 + log.debug("Writing %s to %s" % (network, self.f))
2342 self._write_network(network)
2343
2344 def finish(self):
2345 @@ -206,40 +233,40 @@ class Exporter(object):
2346 # Get all networks that match the family
2347 networks = self.db.search_networks(family=family)
2348
2349 - # Materialise the generator into a list (uses quite some memory)
2350 - networks = list(networks)
2351 + # Create a stack with all networks in order where we can put items back
2352 + # again and retrieve them in the next iteration.
2353 + networks = BufferedStack(networks)
2354
2355 # Walk through all networks
2356 - for i, network in enumerate(networks):
2357 - _network = ipaddress.ip_network("%s" % network)
2358 -
2359 - # Search for all subnets
2360 - subnets = set()
2361 -
2362 - while i < len(networks):
2363 - subnet = networks[i+1]
2364 -
2365 - if subnet.is_subnet_of(network):
2366 - _subnet = ipaddress.ip_network("%s" % subnet)
2367 -
2368 - subnets.add(_subnet)
2369 - subnets.update(_network.address_exclude(_subnet))
2370 -
2371 - i += 1
2372 - else:
2373 + for network in networks:
2374 + # Collect all networks which are a subnet of network
2375 + subnets = []
2376 + for subnet in networks:
2377 + # If the next subnet was not a subnet, we have to push
2378 + # it back on the stack and break this loop
2379 + if not subnet.is_subnet_of(network):
2380 + networks.push(subnet)
2381 break
2382
2383 + subnets.append(subnet)
2384 +
2385 # Write matching countries
2386 - try:
2387 - writers[network.country_code].write(network, subnets)
2388 - except KeyError:
2389 - pass
2390 + if network.country_code and network.country_code in writers:
2391 + # Mismatching subnets
2392 + gaps = [
2393 + subnet for subnet in subnets if not network.country_code == subnet.country_code
2394 + ]
2395 +
2396 + writers[network.country_code].write(network, gaps)
2397
2398 # Write matching ASNs
2399 - try:
2400 - writers[network.asn].write(network, subnets)
2401 - except KeyError:
2402 - pass
2403 + if network.asn and network.asn in writers:
2404 + # Mismatching subnets
2405 + gaps = [
2406 + subnet for subnet in subnets if not network.asn == subnet.asn
2407 + ]
2408 +
2409 + writers[network.asn].write(network, gaps)
2410
2411 # Handle flags
2412 for flag in flags:
2413 @@ -247,10 +274,19 @@ class Exporter(object):
2414 # Fetch the "fake" country code
2415 country = flags[flag]
2416
2417 - try:
2418 - writers[country].write(network, subnets)
2419 - except KeyError:
2420 - pass
2421 + if not country in writers:
2422 + continue
2423 +
2424 + gaps = [
2425 + subnet for subnet in subnets
2426 + if not subnet.has_flag(flag)
2427 + ]
2428 +
2429 + writers[country].write(network, gaps)
2430 +
2431 + # Push all subnets back onto the stack
2432 + for subnet in reversed(subnets):
2433 + networks.push(subnet)
2434
2435 # Write everything to the filesystem
2436 for writer in writers.values():
2437 @@ -262,3 +298,33 @@ class Exporter(object):
2438 )
2439
2440 return os.path.join(directory, filename)
2441 +
2442 +
2443 +class BufferedStack(object):
2444 + """
2445 + This class takes an iterator and when being iterated
2446 + over it returns objects from that iterator for as long
2447 + as there are any.
2448 +
2449 + It additionally has a function to put an item back on
2450 + the back so that it will be returned again at the next
2451 + iteration.
2452 + """
2453 + def __init__(self, iterator):
2454 + self.iterator = iterator
2455 + self.stack = []
2456 +
2457 + def __iter__(self):
2458 + return self
2459 +
2460 + def __next__(self):
2461 + if self.stack:
2462 + return self.stack.pop(0)
2463 +
2464 + return next(self.iterator)
2465 +
2466 + def push(self, elem):
2467 + """
2468 + Takes an element and puts it on the stack
2469 + """
2470 + self.stack.insert(0, elem)
2471 --
2472 2.20.1
2473
2474 From e99a72265c1ba2194b61663eda7e9f14e0083016 Mon Sep 17 00:00:00 2001
2475 From: Michael Tremer <michael.tremer@ipfire.org>
2476 Date: Wed, 28 Oct 2020 09:52:36 +0000
2477 Subject: [PATCH 28/70] location: Fix Python syntax error in verify()
2478
2479 The database is now being opened before the requested
2480 method is called and handle_verify() wasn't updated.
2481
2482 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2483 ---
2484 src/python/location.in | 8 +-------
2485 1 file changed, 1 insertion(+), 7 deletions(-)
2486
2487 diff --git a/src/python/location.in b/src/python/location.in
2488 index 44ad726..b5e5758 100644
2489 --- a/src/python/location.in
2490 +++ b/src/python/location.in
2491 @@ -453,13 +453,7 @@ class CLI(object):
2492
2493 return 0
2494
2495 - def handle_verify(self, ns):
2496 - try:
2497 - db = location.Database(ns.database)
2498 - except FileNotFoundError as e:
2499 - log.error("%s: %s" % (ns.database, e))
2500 - return 127
2501 -
2502 + def handle_verify(self, db, ns):
2503 # Verify the database
2504 with open(ns.public_key, "r") as f:
2505 if not db.verify(f):
2506 --
2507 2.20.1
2508
2509 From 0c74f6b1a3bdce5ebdc2ee452b9baf3e421dd3d1 Mon Sep 17 00:00:00 2001
2510 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2511 Date: Thu, 29 Oct 2020 07:25:53 -0700
2512 Subject: [PATCH 29/70] location update: Remove double conversion of timestamps
2513 MIME-Version: 1.0
2514 Content-Type: text/plain; charset=UTF-8
2515 Content-Transfer-Encoding: 8bit
2516
2517 This caused that the timestamp in DNS was misinterpreted
2518 as local time and often, databases could not be downloaded.
2519
2520 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2521 ---
2522 src/python/location.in | 5 +----
2523 1 file changed, 1 insertion(+), 4 deletions(-)
2524
2525 diff --git a/src/python/location.in b/src/python/location.in
2526 index b5e5758..070640c 100644
2527 --- a/src/python/location.in
2528 +++ b/src/python/location.in
2529 @@ -421,11 +421,8 @@ class CLI(object):
2530 # Fetch the timestamp we need from DNS
2531 t = location.discover_latest_version()
2532
2533 - # Parse timestamp into datetime format
2534 - timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
2535 -
2536 # Check the version of the local database
2537 - if db and timestamp and db.created_at >= timestamp.timestamp():
2538 + if db and t and db.created_at >= t:
2539 log.info("Already on the latest version")
2540 return
2541
2542 --
2543 2.20.1
2544
2545 From 60c1ac0307312614bd6980d30b44bb59b5a6ab6e Mon Sep 17 00:00:00 2001
2546 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2547 Date: Thu, 29 Oct 2020 07:36:46 -0700
2548 Subject: [PATCH 30/70] location.in: do not confuse UTC with local time zones
2549 MIME-Version: 1.0
2550 Content-Type: text/plain; charset=UTF-8
2551 Content-Transfer-Encoding: 8bit
2552
2553 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2554 ---
2555 src/python/location.in | 12 +++++-------
2556 1 file changed, 5 insertions(+), 7 deletions(-)
2557
2558 diff --git a/src/python/location.in b/src/python/location.in
2559 index 070640c..0d09210 100644
2560 --- a/src/python/location.in
2561 +++ b/src/python/location.in
2562 @@ -398,10 +398,7 @@ class CLI(object):
2563
2564 def handle_update(self, db, ns):
2565 if ns.cron and db:
2566 - now = datetime.datetime.utcnow()
2567 -
2568 - # Parse the database timestamp
2569 - t = datetime.datetime.utcfromtimestamp(db.created_at)
2570 + now = time.time()
2571
2572 if ns.cron == "daily":
2573 delta = datetime.timedelta(days=1)
2574 @@ -410,11 +407,12 @@ class CLI(object):
2575 elif ns.cron == "monthly":
2576 delta = datetime.timedelta(days=30)
2577
2578 + delta = delta.total_seconds()
2579 +
2580 # Check if the database has recently been updated
2581 - if t >= (now - delta):
2582 + if db.created_at >= (now - delta):
2583 log.info(
2584 - _("The database has been updated recently (%s)") % \
2585 - format_timedelta(now - t),
2586 + _("The database has been updated recently"),
2587 )
2588 return 3
2589
2590 --
2591 2.20.1
2592
2593 From e7d612e5219ef9ba612ed404e4e2c174110d3dd7 Mon Sep 17 00:00:00 2001
2594 From: =?UTF-8?q?Peter=20M=C3=BCller?= <peter.mueller@ipfire.org>
2595 Date: Tue, 3 Nov 2020 15:31:08 +0000
2596 Subject: [PATCH 31/70] location-importer.in: always convert organisation
2597 handles into upper cases
2598 MIME-Version: 1.0
2599 Content-Type: text/plain; charset=UTF-8
2600 Content-Transfer-Encoding: 8bit
2601
2602 Fixes: #12523
2603
2604 Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
2605 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2606 ---
2607 src/python/location-importer.in | 6 ++++--
2608 1 file changed, 4 insertions(+), 2 deletions(-)
2609
2610 diff --git a/src/python/location-importer.in b/src/python/location-importer.in
2611 index 864eab1..2dec89e 100644
2612 --- a/src/python/location-importer.in
2613 +++ b/src/python/location-importer.in
2614 @@ -560,7 +560,7 @@ class CLI(object):
2615 autnum["asn"] = m.group(2)
2616
2617 elif key == "org":
2618 - autnum[key] = val
2619 + autnum[key] = val.upper()
2620
2621 # Skip empty objects
2622 if not autnum:
2623 @@ -646,7 +646,9 @@ class CLI(object):
2624 # Split line
2625 key, val = split_line(line)
2626
2627 - if key in ("organisation", "org-name"):
2628 + if key == "organisation":
2629 + org[key] = val.upper()
2630 + elif key == "org-name":
2631 org[key] = val
2632
2633 # Skip empty objects
2634 --
2635 2.20.1
2636
2637 From e96704f43acca1a8f56d9a680cce281f5e587ec5 Mon Sep 17 00:00:00 2001
2638 From: Michael Tremer <michael.tremer@ipfire.org>
2639 Date: Wed, 11 Nov 2020 21:16:45 +0000
2640 Subject: [PATCH 32/70] test: Add tests for database enumerator
2641
2642 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2643 ---
2644 src/test-database.c | 53 +++++++++++++++++++++++++++++++++++++++++++++
2645 1 file changed, 53 insertions(+)
2646
2647 diff --git a/src/test-database.c b/src/test-database.c
2648 index b4a75c4..4aef94e 100644
2649 --- a/src/test-database.c
2650 +++ b/src/test-database.c
2651 @@ -38,6 +38,14 @@ const char* DESCRIPTION =
2652 "Maecenas ut venenatis nunc.";
2653 const char* LICENSE = "CC";
2654
2655 +const char* networks[] = {
2656 + "2001:db8::/32",
2657 + "2001:db8:1000::/48",
2658 + "2001:db8:2000::/48",
2659 + "2001:db8:2020::/48",
2660 + NULL,
2661 +};
2662 +
2663 static int attempt_to_open(struct loc_ctx* ctx, char* path) {
2664 FILE* f = fopen(path, "r");
2665 if (!f)
2666 @@ -139,6 +147,24 @@ int main(int argc, char** argv) {
2667 exit(EXIT_FAILURE);
2668 }
2669
2670 + struct loc_network* network = NULL;
2671 +
2672 + // Add some networks
2673 + const char** n = networks;
2674 + while (*n) {
2675 + err = loc_writer_add_network(writer, &network, *n);
2676 + if (err) {
2677 + fprintf(stderr, "Could not add network %s\n", *n);
2678 + exit(EXIT_FAILURE);
2679 + }
2680 +
2681 + // Set a country
2682 + loc_network_set_country_code(network, "XX");
2683 +
2684 + // Next one
2685 + n++;
2686 + }
2687 +
2688 FILE* f = tmpfile();
2689 if (!f) {
2690 fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
2691 @@ -170,6 +196,33 @@ int main(int argc, char** argv) {
2692 exit(EXIT_FAILURE);
2693 }
2694
2695 + // Enumerator
2696 + struct loc_database_enumerator* enumerator;
2697 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS);
2698 + if (err) {
2699 + fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
2700 + exit(EXIT_FAILURE);
2701 + }
2702 +
2703 + // Walk through all networks
2704 + while (1) {
2705 + err = loc_database_enumerator_next_network(enumerator, &network);
2706 + if (err) {
2707 + fprintf(stderr, "Error fetching the next network: %d\n", err);
2708 + exit(EXIT_FAILURE);
2709 + }
2710 +
2711 + if (!network)
2712 + break;
2713 +
2714 + char* s = loc_network_str(network);
2715 + printf("Got network: %s\n", s);
2716 + free(s);
2717 + }
2718 +
2719 + // Free the enumerator
2720 + loc_database_enumerator_unref(enumerator);
2721 +
2722 // Close the database
2723 loc_database_unref(db);
2724 loc_unref(ctx);
2725 --
2726 2.20.1
2727
2728 From ecce288da39a2c0eb60076050ca21e9619f61844 Mon Sep 17 00:00:00 2001
2729 From: Michael Tremer <michael.tremer@ipfire.org>
2730 Date: Wed, 11 Nov 2020 23:01:19 +0000
2731 Subject: [PATCH 33/70] networks: Add list to manage groups of networks
2732
2733 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2734 ---
2735 src/libloc.sym | 11 ++++++
2736 src/loc/network.h | 12 ++++++
2737 src/network.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++
2738 3 files changed, 119 insertions(+)
2739
2740 diff --git a/src/libloc.sym b/src/libloc.sym
2741 index b8296eb..6ef2d27 100644
2742 --- a/src/libloc.sym
2743 +++ b/src/libloc.sym
2744 @@ -98,6 +98,17 @@ global:
2745 loc_network_str;
2746 loc_network_unref;
2747
2748 + # Network List
2749 + loc_network_list_clear;
2750 + loc_network_list_empty;
2751 + loc_network_list_get;
2752 + loc_network_list_new;
2753 + loc_network_list_pop;
2754 + loc_network_list_push;
2755 + loc_network_list_ref;
2756 + loc_network_list_size;
2757 + loc_network_list_unref;
2758 +
2759 # Writer
2760 loc_writer_add_as;
2761 loc_writer_add_country;
2762 diff --git a/src/loc/network.h b/src/loc/network.h
2763 index 70c3803..fd20812 100644
2764 --- a/src/loc/network.h
2765 +++ b/src/loc/network.h
2766 @@ -56,6 +56,18 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
2767
2768 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
2769
2770 +// List
2771 +struct loc_network_list;
2772 +int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
2773 +struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
2774 +struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
2775 +size_t loc_network_list_size(struct loc_network_list* list);
2776 +int loc_network_list_empty(struct loc_network_list* list);
2777 +void loc_network_list_clear(struct loc_network_list* list);
2778 +struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
2779 +int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
2780 +struct loc_network* loc_network_list_pop(struct loc_network_list* list);
2781 +
2782 #ifdef LIBLOC_PRIVATE
2783
2784 int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj);
2785 diff --git a/src/network.c b/src/network.c
2786 index d7b1645..c9e7979 100644
2787 --- a/src/network.c
2788 +++ b/src/network.c
2789 @@ -746,3 +746,99 @@ LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node)
2790 LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
2791 return loc_network_ref(node->network);
2792 }
2793 +
2794 +// List
2795 +
2796 +struct loc_network_list {
2797 + struct loc_ctx* ctx;
2798 + int refcount;
2799 +
2800 + struct loc_network* list[1024];
2801 + size_t size;
2802 + size_t max_size;
2803 +};
2804 +
2805 +LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
2806 + struct loc_network_list** list) {
2807 + struct loc_network_list* l = calloc(1, sizeof(*l));
2808 + if (!l)
2809 + return -ENOMEM;
2810 +
2811 + l->ctx = loc_ref(ctx);
2812 + l->refcount = 1;
2813 +
2814 + // Do not allow this list to grow larger than this
2815 + l->max_size = 1024;
2816 +
2817 + DEBUG(l->ctx, "Network list allocated at %p\n", l);
2818 + *list = l;
2819 + return 0;
2820 +}
2821 +
2822 +LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
2823 + list->refcount++;
2824 +
2825 + return list;
2826 +}
2827 +
2828 +static void loc_network_list_free(struct loc_network_list* list) {
2829 + DEBUG(list->ctx, "Releasing network list at %p\n", list);
2830 +
2831 + for (unsigned int i = 0; i < list->size; i++)
2832 + loc_network_unref(list->list[i]);
2833 +
2834 + loc_unref(list->ctx);
2835 + free(list);
2836 +}
2837 +
2838 +LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
2839 + if (!list)
2840 + return NULL;
2841 +
2842 + if (--list->refcount > 0)
2843 + return list;
2844 +
2845 + loc_network_list_free(list);
2846 + return NULL;
2847 +}
2848 +
2849 +LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
2850 + return list->size;
2851 +}
2852 +
2853 +LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
2854 + return list->size == 0;
2855 +}
2856 +
2857 +LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
2858 + for (unsigned int i = 0; i < list->size; i++)
2859 + loc_network_unref(list->list[i]);
2860 +
2861 + list->size = 0;
2862 +}
2863 +
2864 +LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
2865 + // Check index
2866 + if (index >= list->size)
2867 + return NULL;
2868 +
2869 + return loc_network_ref(list->list[index]);
2870 +}
2871 +
2872 +LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
2873 + // Check if we have space left
2874 + if (list->size == list->max_size)
2875 + return -ENOMEM;
2876 +
2877 + list->list[list->size++] = loc_network_ref(network);
2878 +
2879 + return 0;
2880 +}
2881 +
2882 +LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
2883 + // Return nothing when empty
2884 + if (loc_network_list_empty(list))
2885 + return NULL;
2886 +
2887 + return list->list[list->size--];
2888 +}
2889 --
2890 2.20.1
2891
2892 From 8b2205272b7872a1458ad87811abf58609f38ad4 Mon Sep 17 00:00:00 2001
2893 From: Michael Tremer <michael.tremer@ipfire.org>
2894 Date: Thu, 12 Nov 2020 13:57:35 +0000
2895 Subject: [PATCH 34/70] networks: Add function to dump lists
2896
2897 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2898 ---
2899 src/libloc.sym | 1 +
2900 src/loc/network.h | 1 +
2901 src/network.c | 14 ++++++++++++++
2902 3 files changed, 16 insertions(+)
2903
2904 diff --git a/src/libloc.sym b/src/libloc.sym
2905 index 6ef2d27..a5641c6 100644
2906 --- a/src/libloc.sym
2907 +++ b/src/libloc.sym
2908 @@ -100,6 +100,7 @@ global:
2909
2910 # Network List
2911 loc_network_list_clear;
2912 + loc_network_list_dump;
2913 loc_network_list_empty;
2914 loc_network_list_get;
2915 loc_network_list_new;
2916 diff --git a/src/loc/network.h b/src/loc/network.h
2917 index fd20812..44c50a4 100644
2918 --- a/src/loc/network.h
2919 +++ b/src/loc/network.h
2920 @@ -64,6 +64,7 @@ struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
2921 size_t loc_network_list_size(struct loc_network_list* list);
2922 int loc_network_list_empty(struct loc_network_list* list);
2923 void loc_network_list_clear(struct loc_network_list* list);
2924 +void loc_network_list_dump(struct loc_network_list* list);
2925 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
2926 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
2927 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
2928 diff --git a/src/network.c b/src/network.c
2929 index c9e7979..0977406 100644
2930 --- a/src/network.c
2931 +++ b/src/network.c
2932 @@ -817,6 +817,20 @@ LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
2933 list->size = 0;
2934 }
2935
2936 +LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
2937 + struct loc_network* network;
2938 + char* s;
2939 +
2940 + for (unsigned int i = 0; i < list->size; i++) {
2941 + network = list->list[i];
2942 +
2943 + s = loc_network_str(network);
2944 +
2945 + INFO(list->ctx, "%s\n", s);
2946 + free(s);
2947 + }
2948 +}
2949 +
2950 LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
2951 // Check index
2952 if (index >= list->size)
2953 --
2954 2.20.1
2955
2956 From 850e75167e8e03fe8b951992c9f7bc2ccb9fb711 Mon Sep 17 00:00:00 2001
2957 From: Michael Tremer <michael.tremer@ipfire.org>
2958 Date: Thu, 12 Nov 2020 14:18:40 +0000
2959 Subject: [PATCH 35/70] network: Add functions to break network into subnets
2960
2961 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
2962 ---
2963 src/libloc.sym | 5 +
2964 src/loc/network.h | 6 +
2965 src/network.c | 290 ++++++++++++++++++++++++++++++++++++++++++++-
2966 src/test-network.c | 61 ++++++++++
2967 4 files changed, 361 insertions(+), 1 deletion(-)
2968
2969 diff --git a/src/libloc.sym b/src/libloc.sym
2970 index a5641c6..fcb9ea5 100644
2971 --- a/src/libloc.sym
2972 +++ b/src/libloc.sym
2973 @@ -80,6 +80,8 @@ global:
2974
2975 # Network
2976 loc_network_address_family;
2977 + loc_network_eq;
2978 + loc_network_exclude;
2979 loc_network_format_first_address;
2980 loc_network_format_last_address;
2981 loc_network_get_asn;
2982 @@ -96,6 +98,7 @@ global:
2983 loc_network_set_country_code;
2984 loc_network_set_flag;
2985 loc_network_str;
2986 + loc_network_subnets;
2987 loc_network_unref;
2988
2989 # Network List
2990 @@ -107,7 +110,9 @@ global:
2991 loc_network_list_pop;
2992 loc_network_list_push;
2993 loc_network_list_ref;
2994 + loc_network_list_reverse;
2995 loc_network_list_size;
2996 + loc_network_list_sort;
2997 loc_network_list_unref;
2998
2999 # Writer
3000 diff --git a/src/loc/network.h b/src/loc/network.h
3001 index 44c50a4..4e51a62 100644
3002 --- a/src/loc/network.h
3003 +++ b/src/loc/network.h
3004 @@ -54,7 +54,11 @@ int loc_network_has_flag(struct loc_network* network, uint32_t flag);
3005 int loc_network_set_flag(struct loc_network* network, uint32_t flag);
3006 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
3007
3008 +int loc_network_eq(struct loc_network* self, struct loc_network* other);
3009 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
3010 +struct loc_network_list* loc_network_subnets(struct loc_network* network);
3011 +struct loc_network_list* loc_network_exclude(
3012 + struct loc_network* self, struct loc_network* other);
3013
3014 // List
3015 struct loc_network_list;
3016 @@ -68,6 +72,8 @@ void loc_network_list_dump(struct loc_network_list* list);
3017 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
3018 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
3019 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
3020 +void loc_network_list_sort(struct loc_network_list* list);
3021 +void loc_network_list_reverse(struct loc_network_list* list);
3022
3023 #ifdef LIBLOC_PRIVATE
3024
3025 diff --git a/src/network.c b/src/network.c
3026 index 0977406..6c08070 100644
3027 --- a/src/network.c
3028 +++ b/src/network.c
3029 @@ -97,6 +97,21 @@ static struct in6_addr make_last_address(const struct in6_addr* address, const s
3030 return a;
3031 }
3032
3033 +static struct in6_addr address_increment(const struct in6_addr* address) {
3034 + struct in6_addr a = *address;
3035 +
3036 + for (int octet = 15; octet >= 0; octet--) {
3037 + if (a.s6_addr[octet] < 255) {
3038 + a.s6_addr[octet]++;
3039 + break;
3040 + } else {
3041 + a.s6_addr[octet] = 0;
3042 + }
3043 + }
3044 +
3045 + return a;
3046 +}
3047 +
3048 LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network,
3049 struct in6_addr* address, unsigned int prefix) {
3050 // Address cannot be unspecified
3051 @@ -405,6 +420,69 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
3052 return loc_network_has_flag(network, flag);
3053 }
3054
3055 +LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
3056 +#ifdef ENABLE_DEBUG
3057 + char* n1 = loc_network_str(self);
3058 + char* n2 = loc_network_str(other);
3059 +
3060 + DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2);
3061 +
3062 + free(n1);
3063 + free(n2);
3064 +#endif
3065 +
3066 + // Family must be the same
3067 + if (self->family != other->family) {
3068 + DEBUG(self->ctx, " Family mismatch\n");
3069 +
3070 + return 0;
3071 + }
3072 +
3073 + // The start address must be the same
3074 + if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) {
3075 + DEBUG(self->ctx, " Address mismatch\n");
3076 +
3077 + return 0;
3078 + }
3079 +
3080 + // The prefix length must be the same
3081 + if (self->prefix != other->prefix) {
3082 + DEBUG(self->ctx, " Prefix mismatch\n");
3083 + return 0;
3084 + }
3085 +
3086 + DEBUG(self->ctx, " Yes!\n");
3087 +
3088 + return 1;
3089 +}
3090 +
3091 +static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
3092 + // Families must match
3093 + if (self->family != other->family)
3094 + return -1;
3095 +
3096 + int r = in6_addr_cmp(&self->first_address, &other->first_address);
3097 +
3098 + switch (r) {
3099 + // Smaller
3100 + case -1:
3101 + return 0;
3102 +
3103 + // Larger
3104 + case 1:
3105 + return 1;
3106 +
3107 + default:
3108 + break;
3109 + }
3110 +
3111 + if (self->prefix > other->prefix)
3112 + return 1;
3113 +
3114 + // Dunno
3115 + return 0;
3116 +}
3117 +
3118 LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
3119 // If the start address of the other network is smaller than this network,
3120 // it cannot be a subnet.
3121 @@ -419,6 +497,175 @@ LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_net
3122 return 1;
3123 }
3124
3125 +LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* network) {
3126 + struct loc_network_list* list;
3127 +
3128 + // New prefix length
3129 + unsigned int prefix = network->prefix + 1;
3130 +
3131 + // Check if the new prefix is valid
3132 + if (valid_prefix(&network->first_address, prefix))
3133 + return NULL;
3134 +
3135 + // Create a new list with the result
3136 + int r = loc_network_list_new(network->ctx, &list);
3137 + if (r) {
3138 + ERROR(network->ctx, "Could not create network list: %d\n", r);
3139 + return NULL;
3140 + }
3141 +
3142 + struct loc_network* subnet1 = NULL;
3143 + struct loc_network* subnet2 = NULL;
3144 +
3145 + // Create the first half of the network
3146 + r = loc_network_new(network->ctx, &subnet1, &network->first_address, prefix);
3147 + if (r)
3148 + goto ERROR;
3149 +
3150 + // The next subnet starts after the first one
3151 + struct in6_addr first_address = address_increment(&subnet1->last_address);
3152 +
3153 + // Create the second half of the network
3154 + r = loc_network_new(network->ctx, &subnet2, &first_address, prefix);
3155 + if (r)
3156 + goto ERROR;
3157 +
3158 + // Push the both onto the stack (in reverse order)
3159 + r = loc_network_list_push(list, subnet2);
3160 + if (r)
3161 + goto ERROR;
3162 +
3163 + r = loc_network_list_push(list, subnet1);
3164 + if (r)
3165 + goto ERROR;
3166 +
3167 + loc_network_unref(subnet1);
3168 + loc_network_unref(subnet2);
3169 +
3170 + return list;
3171 +
3172 +ERROR:
3173 + if (subnet1)
3174 + loc_network_unref(subnet1);
3175 +
3176 + if (subnet2)
3177 + loc_network_unref(subnet2);
3178 +
3179 + if (list)
3180 + loc_network_list_unref(list);
3181 +
3182 + return NULL;
3183 +}
3184 +
3185 +LOC_EXPORT struct loc_network_list* loc_network_exclude(
3186 + struct loc_network* self, struct loc_network* other) {
3187 + struct loc_network_list* list;
3188 +
3189 +#ifdef ENABLE_DEBUG
3190 + char* n1 = loc_network_str(self);
3191 + char* n2 = loc_network_str(other);
3192 +
3193 + DEBUG(self->ctx, "Returning %s excluding %s...\n", n1, n2);
3194 +
3195 + free(n1);
3196 + free(n2);
3197 +#endif
3198 +
3199 + // Family must match
3200 + if (self->family != other->family) {
3201 + DEBUG(self->ctx, "Family mismatch\n");
3202 +
3203 + return NULL;
3204 + }
3205 +
3206 + // Other must be a subnet of self
3207 + if (!loc_network_is_subnet_of(other, self)) {
3208 + DEBUG(self->ctx, "Network %p is not contained in network %p\n", other, self);
3209 +
3210 + return NULL;
3211 + }
3212 +
3213 + // We cannot perform this operation if both networks equal
3214 + if (loc_network_eq(self, other)) {
3215 + DEBUG(self->ctx, "Networks %p and %p are equal\n", self, other);
3216 +
3217 + return NULL;
3218 + }
3219 +
3220 + // Create a new list with the result
3221 + int r = loc_network_list_new(self->ctx, &list);
3222 + if (r) {
3223 + ERROR(self->ctx, "Could not create network list: %d\n", r);
3224 + return NULL;
3225 + }
3226 +
3227 + struct loc_network_list* subnets = loc_network_subnets(self);
3228 +
3229 + struct loc_network* subnet1 = NULL;
3230 + struct loc_network* subnet2 = NULL;
3231 +
3232 + while (subnets) {
3233 + // Fetch both subnets
3234 + subnet1 = loc_network_list_get(subnets, 0);
3235 + subnet2 = loc_network_list_get(subnets, 1);
3236 +
3237 + // Free list
3238 + loc_network_list_unref(subnets);
3239 + subnets = NULL;
3240 +
3241 + if (loc_network_eq(other, subnet1)) {
3242 + r = loc_network_list_push(list, subnet2);
3243 + if (r)
3244 + goto ERROR;
3245 +
3246 + } else if (loc_network_eq(other, subnet2)) {
3247 + r = loc_network_list_push(list, subnet1);
3248 + if (r)
3249 + goto ERROR;
3250 +
3251 + } else if (loc_network_is_subnet_of(other, subnet1)) {
3252 + r = loc_network_list_push(list, subnet2);
3253 + if (r)
3254 + goto ERROR;
3255 +
3256 + subnets = loc_network_subnets(subnet1);
3257 +
3258 + } else if (loc_network_is_subnet_of(other, subnet2)) {
3259 + r = loc_network_list_push(list, subnet1);
3260 + if (r)
3261 + goto ERROR;
3262 +
3263 + subnets = loc_network_subnets(subnet2);
3264 +
3265 + } else {
3266 + ERROR(self->ctx, "We should never get here\n");
3267 + goto ERROR;
3268 + }
3269 +
3270 + loc_network_unref(subnet1);
3271 + loc_network_unref(subnet2);
3272 + }
3273 +
3274 +#ifdef ENABLE_DEBUG
3275 + loc_network_list_dump(list);
3276 +#endif
3277 +
3278 + // Return the result
3279 + return list;
3280 +
3281 +ERROR:
3282 + if (subnet1)
3283 + loc_network_unref(subnet1);
3284 +
3285 + if (subnet2)
3286 + loc_network_unref(subnet2);
3287 +
3288 + if (list)
3289 + loc_network_list_unref(list);
3290 +
3291 + return NULL;
3292 +}
3293 +
3294 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
3295 // Add country code
3296 loc_country_code_copy(dbobj->country_code, network->country_code);
3297 @@ -854,5 +1101,46 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
3298 if (loc_network_list_empty(list))
3299 return NULL;
3300
3301 - return list->list[list->size--];
3302 + return list->list[--list->size];
3303 +}
3304 +
3305 +static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
3306 + // Do nothing for invalid indices
3307 + if (i1 >= list->size || i2 >= list->size)
3308 + return;
3309 +
3310 + DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2);
3311 +
3312 + struct loc_network* network1 = list->list[i1];
3313 + struct loc_network* network2 = list->list[i2];
3314 +
3315 + list->list[i1] = network2;
3316 + list->list[i2] = network1;
3317 +}
3318 +
3319 +LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
3320 + unsigned int i = 0;
3321 + unsigned int j = list->size - 1;
3322 +
3323 + while (i < j) {
3324 + loc_network_list_swap(list, i++, j--);
3325 + }
3326 +}
3327 +
3328 +LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
3329 + unsigned int n = list->size;
3330 + int swapped;
3331 +
3332 + do {
3333 + swapped = 0;
3334 +
3335 + for (unsigned int i = 1; i < n; i++) {
3336 + if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
3337 + loc_network_list_swap(list, i-1, i);
3338 + swapped = 1;
3339 + }
3340 + }
3341 +
3342 + n--;
3343 + } while (swapped);
3344 }
3345 diff --git a/src/test-network.c b/src/test-network.c
3346 index b6776b4..af1b2e6 100644
3347 --- a/src/test-network.c
3348 +++ b/src/test-network.c
3349 @@ -118,6 +118,19 @@ int main(int argc, char** argv) {
3350 size_t nodes = loc_network_tree_count_nodes(tree);
3351 printf("The tree has %zu nodes\n", nodes);
3352
3353 + // Check equals function
3354 + err = loc_network_eq(network1, network1);
3355 + if (!err) {
3356 + fprintf(stderr, "Network is not equal with itself\n");
3357 + exit(EXIT_FAILURE);
3358 + }
3359 +
3360 + err = loc_network_eq(network1, network2);
3361 + if (err) {
3362 + fprintf(stderr, "Networks equal unexpectedly\n");
3363 + exit(EXIT_FAILURE);
3364 + }
3365 +
3366 // Check subnet function
3367 err = loc_network_is_subnet_of(network1, network2);
3368 if (err != 0) {
3369 @@ -131,6 +144,54 @@ int main(int argc, char** argv) {
3370 exit(EXIT_FAILURE);
3371 }
3372
3373 + // Make list of subnets
3374 + struct loc_network_list* subnets = loc_network_subnets(network1);
3375 + if (!subnets) {
3376 + fprintf(stderr, "Could not find subnets of network\n");
3377 + exit(EXIT_FAILURE);
3378 + }
3379 +
3380 + loc_network_list_dump(subnets);
3381 +
3382 + while (!loc_network_list_empty(subnets)) {
3383 + struct loc_network* subnet = loc_network_list_pop(subnets);
3384 + if (!subnet) {
3385 + fprintf(stderr, "Received an empty subnet\n");
3386 + exit(EXIT_FAILURE);
3387 + }
3388 +
3389 + char* s = loc_network_str(subnet);
3390 + printf("Received subnet %s\n", s);
3391 + free(s);
3392 +
3393 + if (!loc_network_is_subnet_of(subnet, network1)) {
3394 + fprintf(stderr, "Not a subnet\n");
3395 + exit(EXIT_FAILURE);
3396 + }
3397 +
3398 + loc_network_unref(subnet);
3399 + }
3400 +
3401 + loc_network_list_unref(subnets);
3402 +
3403 + struct loc_network_list* excluded = loc_network_exclude(network1, network2);
3404 + if (!excluded) {
3405 + fprintf(stderr, "Could not create excluded list\n");
3406 + exit(EXIT_FAILURE);
3407 + }
3408 +
3409 + loc_network_list_dump(excluded);
3410 +
3411 + // Reverse it
3412 + loc_network_list_reverse(excluded);
3413 + loc_network_list_dump(excluded);
3414 +
3415 + // Sort them and dump them again
3416 + loc_network_list_sort(excluded);
3417 + loc_network_list_dump(excluded);
3418 +
3419 + loc_network_list_unref(excluded);
3420 +
3421 // Create a database
3422 struct loc_writer* writer;
3423 err = loc_writer_new(ctx, &writer, NULL, NULL);
3424 --
3425 2.20.1
3426
3427 From 6159d384c4a98fe45ec52522e2950719e4982d80 Mon Sep 17 00:00:00 2001
3428 From: Michael Tremer <michael.tremer@ipfire.org>
3429 Date: Thu, 12 Nov 2020 14:24:58 +0000
3430 Subject: [PATCH 36/70] networks: Add function to check if two networks overlap
3431
3432 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3433 ---
3434 src/libloc.sym | 1 +
3435 src/loc/network.h | 1 +
3436 src/network.c | 16 ++++++++++++++++
3437 3 files changed, 18 insertions(+)
3438
3439 diff --git a/src/libloc.sym b/src/libloc.sym
3440 index fcb9ea5..0c2835b 100644
3441 --- a/src/libloc.sym
3442 +++ b/src/libloc.sym
3443 @@ -93,6 +93,7 @@ global:
3444 loc_network_match_flag;
3445 loc_network_new;
3446 loc_network_new_from_string;
3447 + loc_network_overlaps;
3448 loc_network_ref;
3449 loc_network_set_asn;
3450 loc_network_set_country_code;
3451 diff --git a/src/loc/network.h b/src/loc/network.h
3452 index 4e51a62..ef13756 100644
3453 --- a/src/loc/network.h
3454 +++ b/src/loc/network.h
3455 @@ -55,6 +55,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
3456 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
3457
3458 int loc_network_eq(struct loc_network* self, struct loc_network* other);
3459 +int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
3460 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
3461 struct loc_network_list* loc_network_subnets(struct loc_network* network);
3462 struct loc_network_list* loc_network_exclude(
3463 diff --git a/src/network.c b/src/network.c
3464 index 6c08070..d826511 100644
3465 --- a/src/network.c
3466 +++ b/src/network.c
3467 @@ -483,6 +483,22 @@ static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
3468 return 0;
3469 }
3470
3471 +LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network* other) {
3472 + if (loc_network_match_address(self, &other->first_address) == 0)
3473 + return 1;
3474 +
3475 + if (loc_network_match_address(self, &other->last_address) == 0)
3476 + return 1;
3477 +
3478 + if (loc_network_match_address(other, &self->first_address) == 0)
3479 + return 1;
3480 +
3481 + if (loc_network_match_address(other, &self->last_address) == 0)
3482 + return 1;
3483 +
3484 + return 0;
3485 +}
3486 +
3487 LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
3488 // If the start address of the other network is smaller than this network,
3489 // it cannot be a subnet.
3490 --
3491 2.20.1
3492
3493 From e52ba21761f27e592040a2793b2a26bbeeeecc05 Mon Sep 17 00:00:00 2001
3494 From: Michael Tremer <michael.tremer@ipfire.org>
3495 Date: Thu, 12 Nov 2020 14:28:15 +0000
3496 Subject: [PATCH 37/70] networks: Add function to check if network is part of a
3497 list
3498
3499 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3500 ---
3501 src/libloc.sym | 1 +
3502 src/loc/network.h | 1 +
3503 src/network.c | 9 +++++++++
3504 3 files changed, 11 insertions(+)
3505
3506 diff --git a/src/libloc.sym b/src/libloc.sym
3507 index 0c2835b..f1b63a2 100644
3508 --- a/src/libloc.sym
3509 +++ b/src/libloc.sym
3510 @@ -104,6 +104,7 @@ global:
3511
3512 # Network List
3513 loc_network_list_clear;
3514 + loc_network_list_contains;
3515 loc_network_list_dump;
3516 loc_network_list_empty;
3517 loc_network_list_get;
3518 diff --git a/src/loc/network.h b/src/loc/network.h
3519 index ef13756..7804512 100644
3520 --- a/src/loc/network.h
3521 +++ b/src/loc/network.h
3522 @@ -73,6 +73,7 @@ void loc_network_list_dump(struct loc_network_list* list);
3523 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
3524 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
3525 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
3526 +int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
3527 void loc_network_list_sort(struct loc_network_list* list);
3528 void loc_network_list_reverse(struct loc_network_list* list);
3529
3530 diff --git a/src/network.c b/src/network.c
3531 index d826511..fcbdc59 100644
3532 --- a/src/network.c
3533 +++ b/src/network.c
3534 @@ -1120,6 +1120,15 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
3535 return list->list[--list->size];
3536 }
3537
3538 +LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
3539 + for (unsigned int i = 0; i < list->size; i++) {
3540 + if (loc_network_eq(list->list[i], network))
3541 + return 1;
3542 + }
3543 +
3544 + return 0;
3545 +}
3546 +
3547 static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
3548 // Do nothing for invalid indices
3549 if (i1 >= list->size || i2 >= list->size)
3550 --
3551 2.20.1
3552
3553 From f802f3a4decf4827ecc8bcabe269ae9f94f7f32d Mon Sep 17 00:00:00 2001
3554 From: Michael Tremer <michael.tremer@ipfire.org>
3555 Date: Thu, 12 Nov 2020 14:33:22 +0000
3556 Subject: [PATCH 38/70] networks: Add function to merge two lists
3557
3558 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3559 ---
3560 src/libloc.sym | 1 +
3561 src/loc/network.h | 1 +
3562 src/network.c | 13 +++++++++++++
3563 3 files changed, 15 insertions(+)
3564
3565 diff --git a/src/libloc.sym b/src/libloc.sym
3566 index f1b63a2..c0b6b1f 100644
3567 --- a/src/libloc.sym
3568 +++ b/src/libloc.sym
3569 @@ -108,6 +108,7 @@ global:
3570 loc_network_list_dump;
3571 loc_network_list_empty;
3572 loc_network_list_get;
3573 + loc_network_list_merge;
3574 loc_network_list_new;
3575 loc_network_list_pop;
3576 loc_network_list_push;
3577 diff --git a/src/loc/network.h b/src/loc/network.h
3578 index 7804512..e30d91c 100644
3579 --- a/src/loc/network.h
3580 +++ b/src/loc/network.h
3581 @@ -76,6 +76,7 @@ struct loc_network* loc_network_list_pop(struct loc_network_list* list);
3582 int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
3583 void loc_network_list_sort(struct loc_network_list* list);
3584 void loc_network_list_reverse(struct loc_network_list* list);
3585 +int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
3586
3587 #ifdef LIBLOC_PRIVATE
3588
3589 diff --git a/src/network.c b/src/network.c
3590 index fcbdc59..541286d 100644
3591 --- a/src/network.c
3592 +++ b/src/network.c
3593 @@ -1169,3 +1169,16 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
3594 n--;
3595 } while (swapped);
3596 }
3597 +
3598 +LOC_EXPORT int loc_network_list_merge(
3599 + struct loc_network_list* self, struct loc_network_list* other) {
3600 + int r;
3601 +
3602 + for (unsigned int i = 0; i < other->size; i++) {
3603 + r = loc_network_list_push(self, other->list[i]);
3604 + if (r)
3605 + return r;
3606 + }
3607 +
3608 + return 0;
3609 +}
3610 --
3611 2.20.1
3612
3613 From 6d22a179dffd08fcf2a44aafb361725ab22486d4 Mon Sep 17 00:00:00 2001
3614 From: Michael Tremer <michael.tremer@ipfire.org>
3615 Date: Thu, 12 Nov 2020 14:35:43 +0000
3616 Subject: [PATCH 39/70] network: Make lists unique
3617
3618 Networks that are in the list won't be added again
3619
3620 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3621 ---
3622 src/network.c | 4 ++++
3623 1 file changed, 4 insertions(+)
3624
3625 diff --git a/src/network.c b/src/network.c
3626 index 541286d..44571b3 100644
3627 --- a/src/network.c
3628 +++ b/src/network.c
3629 @@ -1103,6 +1103,10 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
3630 }
3631
3632 LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
3633 + // Do not add networks that are already on the list
3634 + if (loc_network_list_contains(list, network))
3635 + return 0;
3636 +
3637 // Check if we have space left
3638 if (list->size == list->max_size)
3639 return -ENOMEM;
3640 --
3641 2.20.1
3642
3643 From 681ff05cb7cdf230d38abf09a330a31498e265a4 Mon Sep 17 00:00:00 2001
3644 From: Michael Tremer <michael.tremer@ipfire.org>
3645 Date: Thu, 12 Nov 2020 19:21:13 +0000
3646 Subject: [PATCH 40/70] database: Pass flag to enumerator to flatten output
3647
3648 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3649 ---
3650 src/database.c | 8 +++++++-
3651 src/loc/database.h | 6 +++++-
3652 src/perl/Location.xs | 2 +-
3653 src/python/database.c | 25 ++++++++++++++++++-------
3654 4 files changed, 31 insertions(+), 10 deletions(-)
3655
3656 diff --git a/src/database.c b/src/database.c
3657 index fa1dad0..9baab33 100644
3658 --- a/src/database.c
3659 +++ b/src/database.c
3660 @@ -104,6 +104,9 @@ struct loc_database_enumerator {
3661 enum loc_network_flags flags;
3662 int family;
3663
3664 + // Flatten output?
3665 + int flatten;
3666 +
3667 // Index of the AS we are looking at
3668 unsigned int as_index;
3669
3670 @@ -933,7 +936,7 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
3671 // Enumerator
3672
3673 LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
3674 - struct loc_database* db, enum loc_database_enumerator_mode mode) {
3675 + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
3676 struct loc_database_enumerator* e = calloc(1, sizeof(*e));
3677 if (!e)
3678 return -ENOMEM;
3679 @@ -944,6 +947,9 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
3680 e->mode = mode;
3681 e->refcount = 1;
3682
3683 + // Flatten output?
3684 + e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
3685 +
3686 // Initialise graph search
3687 //e->network_stack[++e->network_stack_depth] = 0;
3688 e->network_stack_depth = 1;
3689 diff --git a/src/loc/database.h b/src/loc/database.h
3690 index 43173dd..14eb5ea 100644
3691 --- a/src/loc/database.h
3692 +++ b/src/loc/database.h
3693 @@ -55,9 +55,13 @@ enum loc_database_enumerator_mode {
3694 LOC_DB_ENUMERATE_COUNTRIES = 3,
3695 };
3696
3697 +enum loc_database_enumerator_flags {
3698 + LOC_DB_ENUMERATOR_FLAGS_FLATTEN = (1 << 0),
3699 +};
3700 +
3701 struct loc_database_enumerator;
3702 int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
3703 - struct loc_database* db, enum loc_database_enumerator_mode mode);
3704 + struct loc_database* db, enum loc_database_enumerator_mode mode, int flags);
3705 struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator);
3706 struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
3707
3708 diff --git a/src/perl/Location.xs b/src/perl/Location.xs
3709 index dcf3f0d..b7676d2 100644
3710 --- a/src/perl/Location.xs
3711 +++ b/src/perl/Location.xs
3712 @@ -125,7 +125,7 @@ database_countries(db)
3713 PPCODE:
3714 // Create Database enumerator
3715 struct loc_database_enumerator* enumerator;
3716 - int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES);
3717 + int err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_COUNTRIES, 0);
3718
3719 if (err) {
3720 croak("Could not create a database enumerator\n");
3721 diff --git a/src/python/database.c b/src/python/database.c
3722 index 1013a58..7f8c2c2 100644
3723 --- a/src/python/database.c
3724 +++ b/src/python/database.c
3725 @@ -207,10 +207,10 @@ static PyObject* new_database_enumerator(PyTypeObject* type, struct loc_database
3726 return (PyObject*)self;
3727 }
3728
3729 -static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what) {
3730 +static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_enumerator_mode what, int flags) {
3731 struct loc_database_enumerator* enumerator;
3732
3733 - int r = loc_database_enumerator_new(&enumerator, self->db, what);
3734 + int r = loc_database_enumerator_new(&enumerator, self->db, what, flags);
3735 if (r) {
3736 PyErr_SetFromErrno(PyExc_SystemError);
3737 return NULL;
3738 @@ -223,7 +223,7 @@ static PyObject* Database_iterate_all(DatabaseObject* self, enum loc_database_en
3739 }
3740
3741 static PyObject* Database_ases(DatabaseObject* self) {
3742 - return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES);
3743 + return Database_iterate_all(self, LOC_DB_ENUMERATE_ASES, 0);
3744 }
3745
3746 static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
3747 @@ -234,7 +234,7 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
3748
3749 struct loc_database_enumerator* enumerator;
3750
3751 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES);
3752 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_ASES, 0);
3753 if (r) {
3754 PyErr_SetFromErrno(PyExc_SystemError);
3755 return NULL;
3756 @@ -250,7 +250,11 @@ static PyObject* Database_search_as(DatabaseObject* self, PyObject* args) {
3757 }
3758
3759 static PyObject* Database_networks(DatabaseObject* self) {
3760 - return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS);
3761 + return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, 0);
3762 +}
3763 +
3764 +static PyObject* Database_networks_flattened(DatabaseObject *self) {
3765 + return Database_iterate_all(self, LOC_DB_ENUMERATE_NETWORKS, LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
3766 }
3767
3768 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
3769 @@ -264,7 +268,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
3770 return NULL;
3771
3772 struct loc_database_enumerator* enumerator;
3773 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS);
3774 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0);
3775 if (r) {
3776 PyErr_SetFromErrno(PyExc_SystemError);
3777 return NULL;
3778 @@ -317,7 +321,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
3779 }
3780
3781 static PyObject* Database_countries(DatabaseObject* self) {
3782 - return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
3783 + return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES, 0);
3784 }
3785
3786 static struct PyMethodDef Database_methods[] = {
3787 @@ -403,6 +407,13 @@ static struct PyGetSetDef Database_getsetters[] = {
3788 NULL,
3789 NULL,
3790 },
3791 + {
3792 + "networks_flattened",
3793 + (getter)Database_networks_flattened,
3794 + NULL,
3795 + NULL,
3796 + NULL,
3797 + },
3798 {
3799 "vendor",
3800 (getter)Database_get_vendor,
3801 --
3802 2.20.1
3803
3804 From f5e50a47e37e9b29d0d2ee9e5a41e5a5fe5aea7f Mon Sep 17 00:00:00 2001
3805 From: Michael Tremer <michael.tremer@ipfire.org>
3806 Date: Thu, 12 Nov 2020 19:21:58 +0000
3807 Subject: [PATCH 41/70] network: Reduce debugging output
3808
3809 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3810 ---
3811 src/network.c | 28 +++-------------------------
3812 1 file changed, 3 insertions(+), 25 deletions(-)
3813
3814 diff --git a/src/network.c b/src/network.c
3815 index 44571b3..f7071a6 100644
3816 --- a/src/network.c
3817 +++ b/src/network.c
3818 @@ -421,37 +421,17 @@ LOC_EXPORT int loc_network_match_flag(struct loc_network* network, uint32_t flag
3819 }
3820
3821 LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* other) {
3822 -#ifdef ENABLE_DEBUG
3823 - char* n1 = loc_network_str(self);
3824 - char* n2 = loc_network_str(other);
3825 -
3826 - DEBUG(self->ctx, "Is %s equal to %s?\n", n1, n2);
3827 -
3828 - free(n1);
3829 - free(n2);
3830 -#endif
3831 -
3832 // Family must be the same
3833 - if (self->family != other->family) {
3834 - DEBUG(self->ctx, " Family mismatch\n");
3835 -
3836 + if (self->family != other->family)
3837 return 0;
3838 - }
3839
3840 // The start address must be the same
3841 - if (in6_addr_cmp(&self->first_address, &other->first_address) != 0) {
3842 - DEBUG(self->ctx, " Address mismatch\n");
3843 -
3844 + if (in6_addr_cmp(&self->first_address, &other->first_address) != 0)
3845 return 0;
3846 - }
3847
3848 // The prefix length must be the same
3849 - if (self->prefix != other->prefix) {
3850 - DEBUG(self->ctx, " Prefix mismatch\n");
3851 + if (self->prefix != other->prefix)
3852 return 0;
3853 - }
3854 -
3855 - DEBUG(self->ctx, " Yes!\n");
3856
3857 return 1;
3858 }
3859 @@ -1138,8 +1118,6 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1
3860 if (i1 >= list->size || i2 >= list->size)
3861 return;
3862
3863 - DEBUG(list->ctx, "Swapping %u with %u\n", i1, i2);
3864 -
3865 struct loc_network* network1 = list->list[i1];
3866 struct loc_network* network2 = list->list[i2];
3867
3868 --
3869 2.20.1
3870
3871 From 037c65d3a07ec6d37ff063f0645adda6b483b407 Mon Sep 17 00:00:00 2001
3872 From: Michael Tremer <michael.tremer@ipfire.org>
3873 Date: Thu, 12 Nov 2020 19:36:38 +0000
3874 Subject: [PATCH 42/70] python: Export networks exclude function
3875
3876 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3877 ---
3878 src/python/network.c | 39 +++++++++++++++++++++++++++++++++++++++
3879 1 file changed, 39 insertions(+)
3880
3881 diff --git a/src/python/network.c b/src/python/network.c
3882 index 5496d1e..11f672b 100644
3883 --- a/src/python/network.c
3884 +++ b/src/python/network.c
3885 @@ -24,6 +24,24 @@
3886 #include "locationmodule.h"
3887 #include "network.h"
3888
3889 +static PyObject* PyList_FromNetworkList(struct loc_network_list* networks) {
3890 + PyObject* list = PyList_New(0);
3891 + if (!networks)
3892 + return list;
3893 +
3894 + while (!loc_network_list_empty(networks)) {
3895 + struct loc_network* network = loc_network_list_pop(networks);
3896 +
3897 + PyObject* n = new_network(&NetworkType, network);
3898 + PyList_Append(list, n);
3899 +
3900 + loc_network_unref(network);
3901 + Py_DECREF(n);
3902 + }
3903 +
3904 + return list;
3905 +}
3906 +
3907 PyObject* new_network(PyTypeObject* type, struct loc_network* network) {
3908 NetworkObject* self = (NetworkObject*)type->tp_alloc(type, 0);
3909 if (self) {
3910 @@ -154,6 +172,21 @@ static PyObject* Network_set_flag(NetworkObject* self, PyObject* args) {
3911 Py_RETURN_NONE;
3912 }
3913
3914 +static PyObject* Network_exclude(NetworkObject* self, PyObject* args) {
3915 + NetworkObject* other = NULL;
3916 +
3917 + if (!PyArg_ParseTuple(args, "O!", &NetworkType, &other))
3918 + return NULL;
3919 +
3920 + struct loc_network_list* list = loc_network_exclude(self->network, other->network);
3921 +
3922 + // Convert to Python objects
3923 + PyObject* obj = PyList_FromNetworkList(list);
3924 + loc_network_list_unref(list);
3925 +
3926 + return obj;
3927 +}
3928 +
3929 static PyObject* Network_is_subnet_of(NetworkObject* self, PyObject* args) {
3930 NetworkObject* other = NULL;
3931
3932 @@ -191,6 +224,12 @@ static PyObject* Network_get_last_address(NetworkObject* self) {
3933 }
3934
3935 static struct PyMethodDef Network_methods[] = {
3936 + {
3937 + "exclude",
3938 + (PyCFunction)Network_exclude,
3939 + METH_VARARGS,
3940 + NULL,
3941 + },
3942 {
3943 "has_flag",
3944 (PyCFunction)Network_has_flag,
3945 --
3946 2.20.1
3947
3948 From 9a7732c8679e805d4d2d55ea4750c5d70ca4bd2c Mon Sep 17 00:00:00 2001
3949 From: Michael Tremer <michael.tremer@ipfire.org>
3950 Date: Thu, 12 Nov 2020 19:59:22 +0000
3951 Subject: [PATCH 43/70] network: Add more debugging output to stacks
3952
3953 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
3954 ---
3955 src/network.c | 16 +++++++++++++---
3956 1 file changed, 13 insertions(+), 3 deletions(-)
3957
3958 diff --git a/src/network.c b/src/network.c
3959 index f7071a6..d41e873 100644
3960 --- a/src/network.c
3961 +++ b/src/network.c
3962 @@ -1088,8 +1088,12 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
3963 return 0;
3964
3965 // Check if we have space left
3966 - if (list->size == list->max_size)
3967 + if (list->size == list->max_size) {
3968 + ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
3969 return -ENOMEM;
3970 + }
3971 +
3972 + DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
3973
3974 list->list[list->size++] = loc_network_ref(network);
3975
3976 @@ -1098,10 +1102,16 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
3977
3978 LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
3979 // Return nothing when empty
3980 - if (loc_network_list_empty(list))
3981 + if (loc_network_list_empty(list)) {
3982 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
3983 return NULL;
3984 + }
3985
3986 - return list->list[--list->size];
3987 + struct loc_network* network = list->list[--list->size];
3988 +
3989 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
3990 +
3991 + return network;
3992 }
3993
3994 LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
3995 --
3996 2.20.1
3997
3998 From 33a051e0435f6e78cc936f26f3b9ee16b7851025 Mon Sep 17 00:00:00 2001
3999 From: Michael Tremer <michael.tremer@ipfire.org>
4000 Date: Thu, 12 Nov 2020 20:00:09 +0000
4001 Subject: [PATCH 44/70] network: Add new subnet function
4002
4003 The old one is too difficult to use in terms of order
4004 of input parameters and return value.
4005
4006 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4007 ---
4008 src/libloc.sym | 1 +
4009 src/loc/network.h | 1 +
4010 src/network.c | 15 +++++++++++++++
4011 3 files changed, 17 insertions(+)
4012
4013 diff --git a/src/libloc.sym b/src/libloc.sym
4014 index c0b6b1f..5392437 100644
4015 --- a/src/libloc.sym
4016 +++ b/src/libloc.sym
4017 @@ -87,6 +87,7 @@ global:
4018 loc_network_get_asn;
4019 loc_network_get_country_code;
4020 loc_network_has_flag;
4021 + loc_network_is_subnet;
4022 loc_network_is_subnet_of;
4023 loc_network_match_asn;
4024 loc_network_match_country_code;
4025 diff --git a/src/loc/network.h b/src/loc/network.h
4026 index e30d91c..2154cdc 100644
4027 --- a/src/loc/network.h
4028 +++ b/src/loc/network.h
4029 @@ -56,6 +56,7 @@ int loc_network_match_flag(struct loc_network* network, uint32_t flag);
4030
4031 int loc_network_eq(struct loc_network* self, struct loc_network* other);
4032 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
4033 +int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
4034 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
4035 struct loc_network_list* loc_network_subnets(struct loc_network* network);
4036 struct loc_network_list* loc_network_exclude(
4037 diff --git a/src/network.c b/src/network.c
4038 index d41e873..5719111 100644
4039 --- a/src/network.c
4040 +++ b/src/network.c
4041 @@ -479,6 +479,21 @@ LOC_EXPORT int loc_network_overlaps(struct loc_network* self, struct loc_network
4042 return 0;
4043 }
4044
4045 +LOC_EXPORT int loc_network_is_subnet(struct loc_network* self, struct loc_network* other) {
4046 + // If the start address of the other network is smaller than this network,
4047 + // it cannot be a subnet.
4048 + if (in6_addr_cmp(&self->first_address, &other->first_address) < 0)
4049 + return 0;
4050 +
4051 + // If the end address of the other network is greater than this network,
4052 + // it cannot be a subnet.
4053 + if (in6_addr_cmp(&self->last_address, &other->last_address) > 0)
4054 + return 0;
4055 +
4056 + return 1;
4057 +}
4058 +
4059 +// XXX DEPRECATED - I find this too difficult to use
4060 LOC_EXPORT int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other) {
4061 // If the start address of the other network is smaller than this network,
4062 // it cannot be a subnet.
4063 --
4064 2.20.1
4065
4066 From add5bb652ba1dad1127f79cb6a0db2d363a6d5e5 Mon Sep 17 00:00:00 2001
4067 From: Michael Tremer <michael.tremer@ipfire.org>
4068 Date: Thu, 12 Nov 2020 20:01:17 +0000
4069 Subject: [PATCH 45/70] network: Add function to exclude multiple networks at
4070 once
4071
4072 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4073 ---
4074 src/libloc.sym | 1 +
4075 src/loc/network.h | 2 ++
4076 src/network.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++
4077 3 files changed, 88 insertions(+)
4078
4079 diff --git a/src/libloc.sym b/src/libloc.sym
4080 index 5392437..bcd11be 100644
4081 --- a/src/libloc.sym
4082 +++ b/src/libloc.sym
4083 @@ -82,6 +82,7 @@ global:
4084 loc_network_address_family;
4085 loc_network_eq;
4086 loc_network_exclude;
4087 + loc_network_exclude_list;
4088 loc_network_format_first_address;
4089 loc_network_format_last_address;
4090 loc_network_get_asn;
4091 diff --git a/src/loc/network.h b/src/loc/network.h
4092 index 2154cdc..40712b9 100644
4093 --- a/src/loc/network.h
4094 +++ b/src/loc/network.h
4095 @@ -61,6 +61,8 @@ int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other
4096 struct loc_network_list* loc_network_subnets(struct loc_network* network);
4097 struct loc_network_list* loc_network_exclude(
4098 struct loc_network* self, struct loc_network* other);
4099 +struct loc_network_list* loc_network_exclude_list(
4100 + struct loc_network* network, struct loc_network_list* list);
4101
4102 // List
4103 struct loc_network_list;
4104 diff --git a/src/network.c b/src/network.c
4105 index 5719111..751e8e5 100644
4106 --- a/src/network.c
4107 +++ b/src/network.c
4108 @@ -677,6 +677,91 @@ ERROR:
4109 return NULL;
4110 }
4111
4112 +LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
4113 + struct loc_network* network, struct loc_network_list* list) {
4114 + struct loc_network_list* to_check;
4115 +
4116 + // Create a new list with all networks to look at
4117 + int r = loc_network_list_new(network->ctx, &to_check);
4118 + if (r)
4119 + return NULL;
4120 +
4121 + struct loc_network* subnet = NULL;
4122 + struct loc_network_list* subnets = NULL;
4123 +
4124 + for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
4125 + subnet = loc_network_list_get(list, i);
4126 +
4127 + // Find all excluded networks
4128 + struct loc_network_list* excluded = loc_network_exclude(network, subnet);
4129 + if (excluded) {
4130 + // Add them all to the "to check" list
4131 + loc_network_list_merge(to_check, excluded);
4132 + loc_network_list_unref(excluded);
4133 + }
4134 +
4135 + // Cleanup
4136 + loc_network_unref(subnet);
4137 + }
4138 +
4139 + r = loc_network_list_new(network->ctx, &subnets);
4140 + if (r) {
4141 + loc_network_list_unref(to_check);
4142 + return NULL;
4143 + }
4144 +
4145 + while (!loc_network_list_empty(to_check)) {
4146 + struct loc_network* subnet_to_check = loc_network_list_pop(to_check);
4147 +
4148 + // Marks whether this subnet passed all checks
4149 + int passed = 1;
4150 +
4151 + for (unsigned int i = 0; i < loc_network_list_size(list); i++) {
4152 + subnet = loc_network_list_get(list, i);
4153 +
4154 + // Drop this subnet if is is already in list
4155 + if (loc_network_eq(subnet_to_check, subnet)) {
4156 + passed = 0;
4157 + loc_network_unref(subnet);
4158 + break;
4159 + }
4160 +
4161 + // Drop this subnet if is a subnet of another subnet
4162 + if (loc_network_is_subnet_of(subnet, subnet_to_check)) {
4163 + passed = 0;
4164 + loc_network_unref(subnet);
4165 + break;
4166 + }
4167 +
4168 + // Break it down if it overlaps
4169 + if (loc_network_overlaps(subnet_to_check, subnet)) {
4170 + passed = 0;
4171 +
4172 + struct loc_network_list* excluded = loc_network_exclude(subnet_to_check, subnet);
4173 + if (excluded) {
4174 + loc_network_list_merge(to_check, excluded);
4175 + loc_network_list_unref(excluded);
4176 + }
4177 +
4178 + loc_network_unref(subnet);
4179 + break;
4180 + }
4181 +
4182 + loc_network_unref(subnet);
4183 + }
4184 +
4185 + if (passed) {
4186 + r = loc_network_list_push(subnets, subnet_to_check);
4187 + }
4188 +
4189 + loc_network_unref(subnet_to_check);
4190 + }
4191 +
4192 + loc_network_list_unref(to_check);
4193 +
4194 + return subnets;
4195 +}
4196 +
4197 LOC_EXPORT int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj) {
4198 // Add country code
4199 loc_country_code_copy(dbobj->country_code, network->country_code);
4200 --
4201 2.20.1
4202
4203 From d87fd7a3d277b4b03222c7d1680e51b3e45e525b Mon Sep 17 00:00:00 2001
4204 From: Michael Tremer <michael.tremer@ipfire.org>
4205 Date: Thu, 12 Nov 2020 20:02:03 +0000
4206 Subject: [PATCH 46/70] database: Add option to return networks flattened
4207
4208 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4209 ---
4210 src/database.c | 174 ++++++++++++++++++++++++++++++++++++++++---------
4211 1 file changed, 143 insertions(+), 31 deletions(-)
4212
4213 diff --git a/src/database.c b/src/database.c
4214 index 9baab33..7a3d1a7 100644
4215 --- a/src/database.c
4216 +++ b/src/database.c
4217 @@ -118,6 +118,9 @@ struct loc_database_enumerator {
4218 struct loc_node_stack network_stack[MAX_STACK_DEPTH];
4219 int network_stack_depth;
4220 unsigned int* networks_visited;
4221 +
4222 + // For subnet search
4223 + struct loc_network_list* stack;
4224 };
4225
4226 static int loc_database_read_magic(struct loc_database* db) {
4227 @@ -935,6 +938,26 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db,
4228
4229 // Enumerator
4230
4231 +static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
4232 + DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
4233 +
4234 + // Release all references
4235 + loc_database_unref(enumerator->db);
4236 + loc_unref(enumerator->ctx);
4237 +
4238 + if (enumerator->string)
4239 + free(enumerator->string);
4240 +
4241 + // Free network search
4242 + free(enumerator->networks_visited);
4243 +
4244 + // Free subnet stack
4245 + if (enumerator->stack)
4246 + loc_network_list_unref(enumerator->stack);
4247 +
4248 + free(enumerator);
4249 +}
4250 +
4251 LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
4252 struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
4253 struct loc_database_enumerator* e = calloc(1, sizeof(*e));
4254 @@ -951,10 +974,16 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum
4255 e->flatten = (flags & LOC_DB_ENUMERATOR_FLAGS_FLATTEN);
4256
4257 // Initialise graph search
4258 - //e->network_stack[++e->network_stack_depth] = 0;
4259 e->network_stack_depth = 1;
4260 e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
4261
4262 + // Allocate stack
4263 + int r = loc_network_list_new(e->ctx, &e->stack);
4264 + if (r) {
4265 + loc_database_enumerator_free(e);
4266 + return r;
4267 + }
4268 +
4269 DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
4270
4271 *enumerator = e;
4272 @@ -967,22 +996,6 @@ LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct lo
4273 return enumerator;
4274 }
4275
4276 -static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
4277 - DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
4278 -
4279 - // Release all references
4280 - loc_database_unref(enumerator->db);
4281 - loc_unref(enumerator->ctx);
4282 -
4283 - if (enumerator->string)
4284 - free(enumerator->string);
4285 -
4286 - // Free network search
4287 - free(enumerator->networks_visited);
4288 -
4289 - free(enumerator);
4290 -}
4291 -
4292 LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) {
4293 if (!enumerator)
4294 return NULL;
4295 @@ -1116,17 +1129,13 @@ static int loc_database_enumerator_stack_push_node(
4296 return 0;
4297 }
4298
4299 -LOC_EXPORT int loc_database_enumerator_next_network(
4300 - struct loc_database_enumerator* enumerator, struct loc_network** network) {
4301 - // Reset network
4302 - *network = NULL;
4303 -
4304 - // Do not do anything if not in network mode
4305 - if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
4306 +static int __loc_database_enumerator_next_network(
4307 + struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
4308 + // Return top element from the stack
4309 + *network = loc_network_list_pop(enumerator->stack);
4310 + if (*network)
4311 return 0;
4312
4313 - int r;
4314 -
4315 DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
4316 enumerator->network_stack_depth);
4317
4318 @@ -1155,7 +1164,7 @@ LOC_EXPORT int loc_database_enumerator_next_network(
4319 enumerator->db->network_nodes_v1 + node->offset;
4320
4321 // Add edges to stack
4322 - r = loc_database_enumerator_stack_push_node(enumerator,
4323 + int r = loc_database_enumerator_stack_push_node(enumerator,
4324 be32toh(n->one), 1, node->depth + 1);
4325
4326 if (r)
4327 @@ -1181,6 +1190,10 @@ LOC_EXPORT int loc_database_enumerator_next_network(
4328 if (r)
4329 return r;
4330
4331 + // Return all networks when the filter is disabled
4332 + if (!filter)
4333 + return 0;
4334 +
4335 // Check if we are interested in this network
4336
4337 // Skip if the family does not match
4338 @@ -1223,12 +1236,111 @@ LOC_EXPORT int loc_database_enumerator_next_network(
4339 }
4340
4341 // Reached the end of the search
4342 + return 0;
4343 +}
4344
4345 - // Mark all nodes as non-visited
4346 - for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++)
4347 - enumerator->networks_visited[i] = 0;
4348 +static int __loc_database_enumerator_next_network_flattened(
4349 + struct loc_database_enumerator* enumerator, struct loc_network** network) {
4350 + // Fetch the next network
4351 + int r = __loc_database_enumerator_next_network(enumerator, network, 1);
4352 + if (r)
4353 + return r;
4354
4355 - return 0;
4356 + // End if we could not read another network
4357 + if (!*network)
4358 + return 0;
4359 +
4360 + struct loc_network* subnet = NULL;
4361 + struct loc_network_list* subnets;
4362 +
4363 + // Create a list with all subnets
4364 + r = loc_network_list_new(enumerator->ctx, &subnets);
4365 + if (r)
4366 + return r;
4367 +
4368 + // Search all subnets from the database
4369 + while (1) {
4370 + // Fetch the next network in line
4371 + r = __loc_database_enumerator_next_network(enumerator, &subnet, 0);
4372 + if (r)
4373 + goto END;
4374 +
4375 + // End if we did not receive another subnet
4376 + if (!subnet)
4377 + break;
4378 +
4379 + // Collect all subnets in a list
4380 + if (loc_network_is_subnet(*network, subnet)) {
4381 + r = loc_network_list_push(subnets, subnet);
4382 + if (r)
4383 + goto END;
4384 +
4385 + loc_network_unref(subnet);
4386 + continue;
4387 + }
4388 +
4389 + // If this is not a subnet, we push it back onto the stack and break
4390 + r = loc_network_list_push(enumerator->stack, subnet);
4391 + if (r)
4392 + goto END;
4393 +
4394 + loc_network_unref(subnet);
4395 + break;
4396 + }
4397 +
4398 + DEBUG(enumerator->ctx, "Found %zu subnet(s)\n", loc_network_list_size(subnets));
4399 +
4400 + // We can abort here if the network has no subnets
4401 + if (loc_network_list_empty(subnets)) {
4402 + loc_network_list_unref(subnets);
4403 +
4404 + return 0;
4405 + }
4406 +
4407 + // If the network has any subnets, we will break it into smaller parts
4408 + // without the subnets.
4409 + struct loc_network_list* excluded = loc_network_exclude_list(*network, subnets);
4410 + if (!excluded || loc_network_list_empty(excluded)) {
4411 + r = 1;
4412 + goto END;
4413 + }
4414 +
4415 + // Sort the result
4416 + loc_network_list_sort(excluded);
4417 +
4418 + // Reverse the list
4419 + loc_network_list_reverse(excluded);
4420 +
4421 + // Replace network with the first one
4422 + loc_network_unref(*network);
4423 +
4424 + *network = loc_network_list_pop(excluded);
4425 +
4426 + // Push the rest onto the stack
4427 + loc_network_list_merge(enumerator->stack, excluded);
4428 +
4429 + loc_network_list_unref(excluded);
4430 +
4431 +END:
4432 + if (subnet)
4433 + loc_network_unref(subnet);
4434 +
4435 + loc_network_list_unref(subnets);
4436 +
4437 + return r;
4438 +}
4439 +
4440 +LOC_EXPORT int loc_database_enumerator_next_network(
4441 + struct loc_database_enumerator* enumerator, struct loc_network** network) {
4442 + // Do not do anything if not in network mode
4443 + if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
4444 + return 0;
4445 +
4446 + // Flatten output?
4447 + if (enumerator->flatten)
4448 + return __loc_database_enumerator_next_network_flattened(enumerator, network);
4449 +
4450 + return __loc_database_enumerator_next_network(enumerator, network, 1);
4451 }
4452
4453 LOC_EXPORT int loc_database_enumerator_next_country(
4454 --
4455 2.20.1
4456
4457 From d3ae93c27dcd7f6984fdc29cc141621e277f2e2a Mon Sep 17 00:00:00 2001
4458 From: Michael Tremer <michael.tremer@ipfire.org>
4459 Date: Thu, 12 Nov 2020 20:09:20 +0000
4460 Subject: [PATCH 47/70] test: Update API
4461
4462 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4463 ---
4464 src/test-as.c | 2 +-
4465 src/test-database.c | 2 +-
4466 2 files changed, 2 insertions(+), 2 deletions(-)
4467
4468 diff --git a/src/test-as.c b/src/test-as.c
4469 index 839a04c..2d61675 100644
4470 --- a/src/test-as.c
4471 +++ b/src/test-as.c
4472 @@ -95,7 +95,7 @@ int main(int argc, char** argv) {
4473 // Enumerator
4474
4475 struct loc_database_enumerator* enumerator;
4476 - err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES);
4477 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_ASES, 0);
4478 if (err) {
4479 fprintf(stderr, "Could not create a database enumerator\n");
4480 exit(EXIT_FAILURE);
4481 diff --git a/src/test-database.c b/src/test-database.c
4482 index 4aef94e..da4b11c 100644
4483 --- a/src/test-database.c
4484 +++ b/src/test-database.c
4485 @@ -198,7 +198,7 @@ int main(int argc, char** argv) {
4486
4487 // Enumerator
4488 struct loc_database_enumerator* enumerator;
4489 - err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS);
4490 + err = loc_database_enumerator_new(&enumerator, db, LOC_DB_ENUMERATE_NETWORKS, 0);
4491 if (err) {
4492 fprintf(stderr, "Could not initialise the enumerator: %d\n", err);
4493 exit(EXIT_FAILURE);
4494 --
4495 2.20.1
4496
4497 From 594ca328c6e124d0f1eb543e9c8d9bbfe8a7b628 Mon Sep 17 00:00:00 2001
4498 From: Michael Tremer <michael.tremer@ipfire.org>
4499 Date: Thu, 12 Nov 2020 20:09:37 +0000
4500 Subject: [PATCH 48/70] networks: Copy all attributes when splitting networks
4501
4502 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4503 ---
4504 src/network.c | 14 ++++++++++++++
4505 1 file changed, 14 insertions(+)
4506
4507 diff --git a/src/network.c b/src/network.c
4508 index 751e8e5..d67f116 100644
4509 --- a/src/network.c
4510 +++ b/src/network.c
4511 @@ -550,6 +550,20 @@ LOC_EXPORT struct loc_network_list* loc_network_subnets(struct loc_network* netw
4512 if (r)
4513 goto ERROR;
4514
4515 + // Copy country code
4516 + const char* country_code = loc_network_get_country_code(network);
4517 + if (country_code) {
4518 + loc_network_set_country_code(subnet1, country_code);
4519 + loc_network_set_country_code(subnet2, country_code);
4520 + }
4521 +
4522 + // Copy ASN
4523 + uint32_t asn = loc_network_get_asn(network);
4524 + if (asn) {
4525 + loc_network_set_asn(subnet1, asn);
4526 + loc_network_set_asn(subnet2, asn);
4527 + }
4528 +
4529 loc_network_unref(subnet1);
4530 loc_network_unref(subnet2);
4531
4532 --
4533 2.20.1
4534
4535 From 69248038292e9ea1a4ee8912cdfc8700456753ad Mon Sep 17 00:00:00 2001
4536 From: Michael Tremer <michael.tremer@ipfire.org>
4537 Date: Fri, 13 Nov 2020 11:23:33 +0000
4538 Subject: [PATCH 49/70] database: Move network filtering into a separate
4539 function
4540
4541 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4542 ---
4543 src/database.c | 56 +++++++++++++++++++++++---------------------------
4544 1 file changed, 26 insertions(+), 30 deletions(-)
4545
4546 diff --git a/src/database.c b/src/database.c
4547 index 7a3d1a7..72bc8eb 100644
4548 --- a/src/database.c
4549 +++ b/src/database.c
4550 @@ -1129,6 +1129,31 @@ static int loc_database_enumerator_stack_push_node(
4551 return 0;
4552 }
4553
4554 +static int loc_database_enumerator_filter_network(
4555 + struct loc_database_enumerator* enumerator, struct loc_network* network) {
4556 + // Skip if the family does not match
4557 + if (enumerator->family && loc_network_address_family(network) != enumerator->family)
4558 + return 1;
4559 +
4560 + // Skip if the country code does not match
4561 + if (*enumerator->country_code &&
4562 + !loc_network_match_country_code(network, enumerator->country_code))
4563 + return 1;
4564 +
4565 + // Skip if the ASN does not match
4566 + if (enumerator->asn &&
4567 + !loc_network_match_asn(network, enumerator->asn))
4568 + return 1;
4569 +
4570 + // Skip if flags do not match
4571 + if (enumerator->flags &&
4572 + !loc_network_match_flag(network, enumerator->flags))
4573 + return 1;
4574 +
4575 + // Do not filter
4576 + return 0;
4577 +}
4578 +
4579 static int __loc_database_enumerator_next_network(
4580 struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
4581 // Return top element from the stack
4582 @@ -1195,36 +1220,7 @@ static int __loc_database_enumerator_next_network(
4583 return 0;
4584
4585 // Check if we are interested in this network
4586 -
4587 - // Skip if the family does not match
4588 - if (enumerator->family && loc_network_address_family(*network) != enumerator->family) {
4589 - loc_network_unref(*network);
4590 - *network = NULL;
4591 -
4592 - continue;
4593 - }
4594 -
4595 - // Skip if the country code does not match
4596 - if (*enumerator->country_code &&
4597 - !loc_network_match_country_code(*network, enumerator->country_code)) {
4598 - loc_network_unref(*network);
4599 - *network = NULL;
4600 -
4601 - continue;
4602 - }
4603 -
4604 - // Skip if the ASN does not match
4605 - if (enumerator->asn &&
4606 - !loc_network_match_asn(*network, enumerator->asn)) {
4607 - loc_network_unref(*network);
4608 - *network = NULL;
4609 -
4610 - continue;
4611 - }
4612 -
4613 - // Skip if flags do not match
4614 - if (enumerator->flags &&
4615 - !loc_network_match_flag(*network, enumerator->flags)) {
4616 + if (loc_database_enumerator_filter_network(enumerator, *network)) {
4617 loc_network_unref(*network);
4618 *network = NULL;
4619
4620 --
4621 2.20.1
4622
4623 From 2113e71bf7b997c82670c5c22cf91aa6442fe6f3 Mon Sep 17 00:00:00 2001
4624 From: Michael Tremer <michael.tremer@ipfire.org>
4625 Date: Fri, 13 Nov 2020 11:29:02 +0000
4626 Subject: [PATCH 50/70] database: Filter results coming from stack
4627
4628 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4629 ---
4630 src/database.c | 18 ++++++++++++++++--
4631 1 file changed, 16 insertions(+), 2 deletions(-)
4632
4633 diff --git a/src/database.c b/src/database.c
4634 index 72bc8eb..0f3cdc2 100644
4635 --- a/src/database.c
4636 +++ b/src/database.c
4637 @@ -1157,9 +1157,23 @@ static int loc_database_enumerator_filter_network(
4638 static int __loc_database_enumerator_next_network(
4639 struct loc_database_enumerator* enumerator, struct loc_network** network, int filter) {
4640 // Return top element from the stack
4641 - *network = loc_network_list_pop(enumerator->stack);
4642 - if (*network)
4643 + while (1) {
4644 + *network = loc_network_list_pop(enumerator->stack);
4645 +
4646 + // Stack is empty
4647 + if (!*network)
4648 + break;
4649 +
4650 + // Throw away any networks by filter
4651 + if (filter && loc_database_enumerator_filter_network(enumerator, *network)) {
4652 + loc_network_unref(*network);
4653 + *network = NULL;
4654 + continue;
4655 + }
4656 +
4657 + // Return result
4658 return 0;
4659 + }
4660
4661 DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
4662 enumerator->network_stack_depth);
4663 --
4664 2.20.1
4665
4666 From d33753688138c9938743dafbbdddf220dd2afd14 Mon Sep 17 00:00:00 2001
4667 From: Michael Tremer <michael.tremer@ipfire.org>
4668 Date: Fri, 13 Nov 2020 11:29:15 +0000
4669 Subject: [PATCH 51/70] network: Sort result of excluded lists
4670
4671 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4672 ---
4673 src/database.c | 3 ---
4674 src/network.c | 3 +++
4675 2 files changed, 3 insertions(+), 3 deletions(-)
4676
4677 diff --git a/src/database.c b/src/database.c
4678 index 0f3cdc2..6849d97 100644
4679 --- a/src/database.c
4680 +++ b/src/database.c
4681 @@ -1315,9 +1315,6 @@ static int __loc_database_enumerator_next_network_flattened(
4682 goto END;
4683 }
4684
4685 - // Sort the result
4686 - loc_network_list_sort(excluded);
4687 -
4688 // Reverse the list
4689 loc_network_list_reverse(excluded);
4690
4691 diff --git a/src/network.c b/src/network.c
4692 index d67f116..9d02bf8 100644
4693 --- a/src/network.c
4694 +++ b/src/network.c
4695 @@ -773,6 +773,9 @@ LOC_EXPORT struct loc_network_list* loc_network_exclude_list(
4696
4697 loc_network_list_unref(to_check);
4698
4699 + // Sort the result
4700 + loc_network_list_sort(subnets);
4701 +
4702 return subnets;
4703 }
4704
4705 --
4706 2.20.1
4707
4708 From 8d777f12f7ffa4df1b28d197563888296803b727 Mon Sep 17 00:00:00 2001
4709 From: Michael Tremer <michael.tremer@ipfire.org>
4710 Date: Fri, 13 Nov 2020 11:38:15 +0000
4711 Subject: [PATCH 52/70] network: Add function to pop first element from stack
4712
4713 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4714 ---
4715 src/database.c | 6 ++----
4716 src/libloc.sym | 1 +
4717 src/loc/network.h | 1 +
4718 src/network.c | 19 +++++++++++++++++++
4719 4 files changed, 23 insertions(+), 4 deletions(-)
4720
4721 diff --git a/src/database.c b/src/database.c
4722 index 6849d97..b9d870f 100644
4723 --- a/src/database.c
4724 +++ b/src/database.c
4725 @@ -1315,15 +1315,13 @@ static int __loc_database_enumerator_next_network_flattened(
4726 goto END;
4727 }
4728
4729 - // Reverse the list
4730 - loc_network_list_reverse(excluded);
4731 -
4732 // Replace network with the first one
4733 loc_network_unref(*network);
4734
4735 - *network = loc_network_list_pop(excluded);
4736 + *network = loc_network_list_pop_first(excluded);
4737
4738 // Push the rest onto the stack
4739 + loc_network_list_reverse(excluded);
4740 loc_network_list_merge(enumerator->stack, excluded);
4741
4742 loc_network_list_unref(excluded);
4743 diff --git a/src/libloc.sym b/src/libloc.sym
4744 index bcd11be..6139db6 100644
4745 --- a/src/libloc.sym
4746 +++ b/src/libloc.sym
4747 @@ -113,6 +113,7 @@ global:
4748 loc_network_list_merge;
4749 loc_network_list_new;
4750 loc_network_list_pop;
4751 + loc_network_list_pop_first;
4752 loc_network_list_push;
4753 loc_network_list_ref;
4754 loc_network_list_reverse;
4755 diff --git a/src/loc/network.h b/src/loc/network.h
4756 index 40712b9..203e61c 100644
4757 --- a/src/loc/network.h
4758 +++ b/src/loc/network.h
4759 @@ -76,6 +76,7 @@ void loc_network_list_dump(struct loc_network_list* list);
4760 struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
4761 int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
4762 struct loc_network* loc_network_list_pop(struct loc_network_list* list);
4763 +struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
4764 int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
4765 void loc_network_list_sort(struct loc_network_list* list);
4766 void loc_network_list_reverse(struct loc_network_list* list);
4767 diff --git a/src/network.c b/src/network.c
4768 index 9d02bf8..e7dc97e 100644
4769 --- a/src/network.c
4770 +++ b/src/network.c
4771 @@ -1231,6 +1231,25 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
4772 return network;
4773 }
4774
4775 +LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
4776 + // Return nothing when empty
4777 + if (loc_network_list_empty(list)) {
4778 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
4779 + return NULL;
4780 + }
4781 +
4782 + struct loc_network* network = list->list[0];
4783 +
4784 + // Move all elements to the top of the stack
4785 + for (unsigned int i = 0; i < --list->size; i++) {
4786 + list->list[i] = list->list[i+1];
4787 + }
4788 +
4789 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
4790 +
4791 + return network;
4792 +}
4793 +
4794 LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
4795 for (unsigned int i = 0; i < list->size; i++) {
4796 if (loc_network_eq(list->list[i], network))
4797 --
4798 2.20.1
4799
4800 From 7933f5bfb4dd7603cb646e192840762bf6394292 Mon Sep 17 00:00:00 2001
4801 From: Michael Tremer <michael.tremer@ipfire.org>
4802 Date: Fri, 13 Nov 2020 11:43:53 +0000
4803 Subject: [PATCH 53/70] network: Unexport all tree functions
4804
4805 These should not be exported
4806
4807 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
4808 ---
4809 src/network.c | 28 ++++++++++++++--------------
4810 src/test-network.c | 9 +++++++++
4811 2 files changed, 23 insertions(+), 14 deletions(-)
4812
4813 diff --git a/src/network.c b/src/network.c
4814 index e7dc97e..d015579 100644
4815 --- a/src/network.c
4816 +++ b/src/network.c
4817 @@ -847,7 +847,7 @@ struct loc_network_tree_node {
4818 struct loc_network* network;
4819 };
4820
4821 -LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
4822 +int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
4823 struct loc_network_tree* t = calloc(1, sizeof(*t));
4824 if (!t)
4825 return -ENOMEM;
4826 @@ -867,7 +867,7 @@ LOC_EXPORT int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree
4827 return 0;
4828 }
4829
4830 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
4831 +struct loc_network_tree_node* loc_network_tree_get_root(struct loc_network_tree* tree) {
4832 return loc_network_tree_node_ref(tree->root);
4833 }
4834
4835 @@ -939,7 +939,7 @@ static int __loc_network_tree_walk(struct loc_ctx* ctx, struct loc_network_tree_
4836 return 0;
4837 }
4838
4839 -LOC_EXPORT int loc_network_tree_walk(struct loc_network_tree* tree,
4840 +int loc_network_tree_walk(struct loc_network_tree* tree,
4841 int(*filter_callback)(struct loc_network* network, void* data),
4842 int(*callback)(struct loc_network* network, void* data), void* data) {
4843 return __loc_network_tree_walk(tree->ctx, tree->root, filter_callback, callback, data);
4844 @@ -954,7 +954,7 @@ static void loc_network_tree_free(struct loc_network_tree* tree) {
4845 free(tree);
4846 }
4847
4848 -LOC_EXPORT struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
4849 +struct loc_network_tree* loc_network_tree_unref(struct loc_network_tree* tree) {
4850 if (--tree->refcount > 0)
4851 return tree;
4852
4853 @@ -975,13 +975,13 @@ static int __loc_network_tree_dump(struct loc_network* network, void* data) {
4854 return 0;
4855 }
4856
4857 -LOC_EXPORT int loc_network_tree_dump(struct loc_network_tree* tree) {
4858 +int loc_network_tree_dump(struct loc_network_tree* tree) {
4859 DEBUG(tree->ctx, "Dumping network tree at %p\n", tree);
4860
4861 return loc_network_tree_walk(tree, NULL, __loc_network_tree_dump, NULL);
4862 }
4863
4864 -LOC_EXPORT int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
4865 +int loc_network_tree_add_network(struct loc_network_tree* tree, struct loc_network* network) {
4866 DEBUG(tree->ctx, "Adding network %p to tree %p\n", network, tree);
4867
4868 struct loc_network_tree_node* node = loc_network_tree_get_path(tree,
4869 @@ -1012,7 +1012,7 @@ static int __loc_network_tree_count(struct loc_network* network, void* data) {
4870 return 0;
4871 }
4872
4873 -LOC_EXPORT size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
4874 +size_t loc_network_tree_count_networks(struct loc_network_tree* tree) {
4875 size_t counter = 0;
4876
4877 int r = loc_network_tree_walk(tree, NULL, __loc_network_tree_count, &counter);
4878 @@ -1034,11 +1034,11 @@ static size_t __loc_network_tree_count_nodes(struct loc_network_tree_node* node)
4879 return counter;
4880 }
4881
4882 -LOC_EXPORT size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
4883 +size_t loc_network_tree_count_nodes(struct loc_network_tree* tree) {
4884 return __loc_network_tree_count_nodes(tree->root);
4885 }
4886
4887 -LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
4888 +int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network_tree_node** node) {
4889 struct loc_network_tree_node* n = calloc(1, sizeof(*n));
4890 if (!n)
4891 return -ENOMEM;
4892 @@ -1053,7 +1053,7 @@ LOC_EXPORT int loc_network_tree_node_new(struct loc_ctx* ctx, struct loc_network
4893 return 0;
4894 }
4895
4896 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
4897 +struct loc_network_tree_node* loc_network_tree_node_ref(struct loc_network_tree_node* node) {
4898 if (node)
4899 node->refcount++;
4900
4901 @@ -1076,7 +1076,7 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) {
4902 free(node);
4903 }
4904
4905 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
4906 +struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
4907 if (!node)
4908 return NULL;
4909
4910 @@ -1087,7 +1087,7 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_
4911 return NULL;
4912 }
4913
4914 -LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
4915 +struct loc_network_tree_node* loc_network_tree_node_get(struct loc_network_tree_node* node, unsigned int index) {
4916 if (index == 0)
4917 node = node->zero;
4918 else
4919 @@ -1099,11 +1099,11 @@ LOC_EXPORT struct loc_network_tree_node* loc_network_tree_node_get(struct loc_ne
4920 return loc_network_tree_node_ref(node);
4921 }
4922
4923 -LOC_EXPORT int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
4924 +int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
4925 return (!!node->network);
4926 }
4927
4928 -LOC_EXPORT struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
4929 +struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
4930 return loc_network_ref(node->network);
4931 }
4932
4933 diff --git a/src/test-network.c b/src/test-network.c
4934 index af1b2e6..7c90224 100644
4935 --- a/src/test-network.c
4936 +++ b/src/test-network.c
4937 @@ -37,12 +37,14 @@ int main(int argc, char** argv) {
4938 // Enable debug logging
4939 loc_set_log_priority(ctx, LOG_DEBUG);
4940
4941 +#if 0
4942 struct loc_network_tree* tree;
4943 err = loc_network_tree_new(ctx, &tree);
4944 if (err) {
4945 fprintf(stderr, "Could not create the network tree\n");
4946 exit(EXIT_FAILURE);
4947 }
4948 +#endif
4949
4950 // Create a network
4951 struct loc_network* network1;
4952 @@ -58,12 +60,14 @@ int main(int argc, char** argv) {
4953 exit(EXIT_FAILURE);
4954 }
4955
4956 +#if 0
4957 // Adding network to the tree
4958 err = loc_network_tree_add_network(tree, network1);
4959 if (err) {
4960 fprintf(stderr, "Could not add network to the tree\n");
4961 exit(EXIT_FAILURE);
4962 }
4963 +#endif
4964
4965 // Check if the first and last addresses are correct
4966 char* string = loc_network_format_first_address(network1);
4967 @@ -101,6 +105,7 @@ int main(int argc, char** argv) {
4968 exit(EXIT_FAILURE);
4969 }
4970
4971 +#if 0
4972 // Adding network to the tree
4973 err = loc_network_tree_add_network(tree, network2);
4974 if (err) {
4975 @@ -117,6 +122,7 @@ int main(int argc, char** argv) {
4976
4977 size_t nodes = loc_network_tree_count_nodes(tree);
4978 printf("The tree has %zu nodes\n", nodes);
4979 +#endif
4980
4981 // Check equals function
4982 err = loc_network_eq(network1, network1);
4983 @@ -260,7 +266,10 @@ int main(int argc, char** argv) {
4984 loc_network_unref(network2);
4985 loc_network_unref(network3);
4986 loc_network_unref(network4);
4987 +
4988 +#if 0
4989 loc_network_tree_unref(tree);
4990 +#endif
4991
4992 // And open it again from disk
4993 struct loc_database* db;
4994 --
4995 2.20.1
4996
4997 From c242f7325bd6fc4ba26047ac24196d1c161c6e01 Mon Sep 17 00:00:00 2001
4998 From: Michael Tremer <michael.tremer@ipfire.org>
4999 Date: Fri, 13 Nov 2020 12:09:03 +0000
5000 Subject: [PATCH 54/70] python: Move tree flattening into C
5001
5002 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
5003 ---
5004 src/python/database.c | 8 ++-
5005 src/python/export.py | 138 +++++-------------------------------------
5006 2 files changed, 21 insertions(+), 125 deletions(-)
5007
5008 diff --git a/src/python/database.c b/src/python/database.c
5009 index 7f8c2c2..d169547 100644
5010 --- a/src/python/database.c
5011 +++ b/src/python/database.c
5012 @@ -258,17 +258,19 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
5013 }
5014
5015 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
5016 - char* kwlist[] = { "country_code", "asn", "flags", "family", NULL };
5017 + char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL };
5018 const char* country_code = NULL;
5019 unsigned int asn = 0;
5020 int flags = 0;
5021 int family = 0;
5022 + int flatten = 0;
5023
5024 - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siii", kwlist, &country_code, &asn, &flags, &family))
5025 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten))
5026 return NULL;
5027
5028 struct loc_database_enumerator* enumerator;
5029 - int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS, 0);
5030 + int r = loc_database_enumerator_new(&enumerator, self->db, LOC_DB_ENUMERATE_NETWORKS,
5031 + (flatten) ? LOC_DB_ENUMERATOR_FLAGS_FLATTEN : 0);
5032 if (r) {
5033 PyErr_SetFromErrno(PyExc_SystemError);
5034 return NULL;
5035 diff --git a/src/python/export.py b/src/python/export.py
5036 index dd44332..be4a68e 100644
5037 --- a/src/python/export.py
5038 +++ b/src/python/export.py
5039 @@ -87,58 +87,12 @@ class OutputWriter(object):
5040 def _write_network(self, network):
5041 self.f.write("%s\n" % network)
5042
5043 - def write(self, network, subnets):
5044 + def write(self, network):
5045 if self.flatten and self._flatten(network):
5046 log.debug("Skipping writing network %s (last one was %s)" % (network, self._last_network))
5047 return
5048
5049 - # Convert network into a Python object
5050 - _network = ipaddress.ip_network("%s" % network)
5051 -
5052 - # Write the network when it has no subnets
5053 - if not subnets:
5054 - log.debug("Writing %s to %s" % (_network, self.f))
5055 - return self._write_network(_network)
5056 -
5057 - # Convert subnets into Python objects
5058 - _subnets = [ipaddress.ip_network("%s" % subnet) for subnet in subnets]
5059 -
5060 - # Split the network into smaller bits so that
5061 - # we can accomodate for any gaps in it later
5062 - to_check = set()
5063 - for _subnet in _subnets:
5064 - to_check.update(
5065 - _network.address_exclude(_subnet)
5066 - )
5067 -
5068 - # Clear the list of all subnets
5069 - subnets = []
5070 -
5071 - # Check if all subnets to not overlap with anything else
5072 - while to_check:
5073 - subnet_to_check = to_check.pop()
5074 -
5075 - for _subnet in _subnets:
5076 - # Drop this subnet if it equals one of the subnets
5077 - # or if it is subnet of one of them
5078 - if subnet_to_check == _subnet or subnet_to_check.subnet_of(_subnet):
5079 - break
5080 -
5081 - # Break it down if it overlaps
5082 - if subnet_to_check.overlaps(_subnet):
5083 - to_check.update(
5084 - subnet_to_check.address_exclude(_subnet)
5085 - )
5086 - break
5087 -
5088 - # Add the subnet again as it passed the check
5089 - else:
5090 - subnets.append(subnet_to_check)
5091 -
5092 - # Write all networks as compact as possible
5093 - for network in ipaddress.collapse_addresses(subnets):
5094 - log.debug("Writing %s to %s" % (network, self.f))
5095 - self._write_network(network)
5096 + return self._write_network(network)
5097
5098 def finish(self):
5099 """
5100 @@ -188,7 +142,7 @@ class XTGeoIPOutputWriter(OutputWriter):
5101 mode = "wb"
5102
5103 def _write_network(self, network):
5104 - for address in (network.network_address, network.broadcast_address):
5105 + for address in (network.first_address, network.last_address):
5106 # Convert this into a string of bits
5107 bytes = socket.inet_pton(
5108 socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
5109 @@ -231,42 +185,21 @@ class Exporter(object):
5110 writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
5111
5112 # Get all networks that match the family
5113 - networks = self.db.search_networks(family=family)
5114 -
5115 - # Create a stack with all networks in order where we can put items back
5116 - # again and retrieve them in the next iteration.
5117 - networks = BufferedStack(networks)
5118 + networks = self.db.search_networks(family=family, flatten=True)
5119
5120 # Walk through all networks
5121 for network in networks:
5122 - # Collect all networks which are a subnet of network
5123 - subnets = []
5124 - for subnet in networks:
5125 - # If the next subnet was not a subnet, we have to push
5126 - # it back on the stack and break this loop
5127 - if not subnet.is_subnet_of(network):
5128 - networks.push(subnet)
5129 - break
5130 -
5131 - subnets.append(subnet)
5132 -
5133 # Write matching countries
5134 - if network.country_code and network.country_code in writers:
5135 - # Mismatching subnets
5136 - gaps = [
5137 - subnet for subnet in subnets if not network.country_code == subnet.country_code
5138 - ]
5139 -
5140 - writers[network.country_code].write(network, gaps)
5141 + try:
5142 + writers[network.country_code].write(network)
5143 + except KeyError:
5144 + pass
5145
5146 # Write matching ASNs
5147 - if network.asn and network.asn in writers:
5148 - # Mismatching subnets
5149 - gaps = [
5150 - subnet for subnet in subnets if not network.asn == subnet.asn
5151 - ]
5152 -
5153 - writers[network.asn].write(network, gaps)
5154 + try:
5155 + writers[network.asn].write(network)
5156 + except KeyError:
5157 + pass
5158
5159 # Handle flags
5160 for flag in flags:
5161 @@ -274,19 +207,10 @@ class Exporter(object):
5162 # Fetch the "fake" country code
5163 country = flags[flag]
5164
5165 - if not country in writers:
5166 - continue
5167 -
5168 - gaps = [
5169 - subnet for subnet in subnets
5170 - if not subnet.has_flag(flag)
5171 - ]
5172 -
5173 - writers[country].write(network, gaps)
5174 -
5175 - # Push all subnets back onto the stack
5176 - for subnet in reversed(subnets):
5177 - networks.push(subnet)
5178 + try:
5179 + writers[country].write(network)
5180 + except KeyError:
5181 + pass
5182
5183 # Write everything to the filesystem
5184 for writer in writers.values():
5185 @@ -298,33 +222,3 @@ class Exporter(object):
5186 )
5187
5188 return os.path.join(directory, filename)
5189 -
5190 -
5191 -class BufferedStack(object):
5192 - """
5193 - This class takes an iterator and when being iterated
5194 - over it returns objects from that iterator for as long
5195 - as there are any.
5196 -
5197 - It additionally has a function to put an item back on
5198 - the back so that it will be returned again at the next
5199 - iteration.
5200 - """
5201 - def __init__(self, iterator):
5202 - self.iterator = iterator
5203 - self.stack = []
5204 -
5205 - def __iter__(self):
5206 - return self
5207 -
5208 - def __next__(self):
5209 - if self.stack:
5210 - return self.stack.pop(0)
5211 -
5212 - return next(self.iterator)
5213 -
5214 - def push(self, elem):
5215 - """
5216 - Takes an element and puts it on the stack
5217 - """
5218 - self.stack.insert(0, elem)
5219 --
5220 2.20.1
5221
5222 From e0b9ff5f38beb0d560b16db881647e5a75127df1 Mon Sep 17 00:00:00 2001
5223 From: Michael Tremer <michael.tremer@ipfire.org>
5224 Date: Sun, 15 Nov 2020 15:02:28 +0000
5225 Subject: [PATCH 55/70] Move network lists into an own file
5226
5227 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
5228 ---
5229 Makefile.am | 2 +
5230 src/libloc.sym | 1 +
5231 src/loc/network-list.h | 37 +++++++
5232 src/loc/network.h | 20 +---
5233 src/network-list.c | 224 +++++++++++++++++++++++++++++++++++++++++
5234 src/network.c | 207 +------------------------------------
5235 src/python/network.c | 1 +
5236 7 files changed, 269 insertions(+), 223 deletions(-)
5237 create mode 100644 src/loc/network-list.h
5238 create mode 100644 src/network-list.c
5239
5240 diff --git a/Makefile.am b/Makefile.am
5241 index a0431a6..f0d8c4c 100644
5242 --- a/Makefile.am
5243 +++ b/Makefile.am
5244 @@ -96,6 +96,7 @@ pkginclude_HEADERS = \
5245 src/loc/database.h \
5246 src/loc/format.h \
5247 src/loc/network.h \
5248 + src/loc/network-list.h \
5249 src/loc/private.h \
5250 src/loc/stringpool.h \
5251 src/loc/resolv.h \
5252 @@ -110,6 +111,7 @@ src_libloc_la_SOURCES = \
5253 src/country.c \
5254 src/database.c \
5255 src/network.c \
5256 + src/network-list.c \
5257 src/resolv.c \
5258 src/stringpool.c \
5259 src/writer.c
5260 diff --git a/src/libloc.sym b/src/libloc.sym
5261 index 6139db6..453a1be 100644
5262 --- a/src/libloc.sym
5263 +++ b/src/libloc.sym
5264 @@ -87,6 +87,7 @@ global:
5265 loc_network_format_last_address;
5266 loc_network_get_asn;
5267 loc_network_get_country_code;
5268 + loc_network_gt;
5269 loc_network_has_flag;
5270 loc_network_is_subnet;
5271 loc_network_is_subnet_of;
5272 diff --git a/src/loc/network-list.h b/src/loc/network-list.h
5273 new file mode 100644
5274 index 0000000..af3b28d
5275 --- /dev/null
5276 +++ b/src/loc/network-list.h
5277 @@ -0,0 +1,37 @@
5278 +/*
5279 + libloc - A library to determine the location of someone on the Internet
5280 +
5281 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
5282 +
5283 + This library is free software; you can redistribute it and/or
5284 + modify it under the terms of the GNU Lesser General Public
5285 + License as published by the Free Software Foundation; either
5286 + version 2.1 of the License, or (at your option) any later version.
5287 +
5288 + This library is distributed in the hope that it will be useful,
5289 + but WITHOUT ANY WARRANTY; without even the implied warranty of
5290 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
5291 + Lesser General Public License for more details.
5292 +*/
5293 +
5294 +#ifndef LIBLOC_NETWORK_LIST_H
5295 +#define LIBLOC_NETWORK_LIST_H
5296 +
5297 +struct loc_network_list;
5298 +int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
5299 +struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
5300 +struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
5301 +size_t loc_network_list_size(struct loc_network_list* list);
5302 +int loc_network_list_empty(struct loc_network_list* list);
5303 +void loc_network_list_clear(struct loc_network_list* list);
5304 +void loc_network_list_dump(struct loc_network_list* list);
5305 +struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
5306 +int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
5307 +struct loc_network* loc_network_list_pop(struct loc_network_list* list);
5308 +struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
5309 +int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
5310 +void loc_network_list_sort(struct loc_network_list* list);
5311 +void loc_network_list_reverse(struct loc_network_list* list);
5312 +int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
5313 +
5314 +#endif
5315 diff --git a/src/loc/network.h b/src/loc/network.h
5316 index 203e61c..d86b685 100644
5317 --- a/src/loc/network.h
5318 +++ b/src/loc/network.h
5319 @@ -21,6 +21,7 @@
5320
5321 #include <loc/libloc.h>
5322 #include <loc/format.h>
5323 +#include <loc/network-list.h>
5324
5325 enum loc_network_flags {
5326 LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1
5327 @@ -55,6 +56,7 @@ int loc_network_set_flag(struct loc_network* network, uint32_t flag);
5328 int loc_network_match_flag(struct loc_network* network, uint32_t flag);
5329
5330 int loc_network_eq(struct loc_network* self, struct loc_network* other);
5331 +int loc_network_gt(struct loc_network* self, struct loc_network* other);
5332 int loc_network_overlaps(struct loc_network* self, struct loc_network* other);
5333 int loc_network_is_subnet(struct loc_network* self, struct loc_network* other);
5334 int loc_network_is_subnet_of(struct loc_network* self, struct loc_network* other);
5335 @@ -64,24 +66,6 @@ struct loc_network_list* loc_network_exclude(
5336 struct loc_network_list* loc_network_exclude_list(
5337 struct loc_network* network, struct loc_network_list* list);
5338
5339 -// List
5340 -struct loc_network_list;
5341 -int loc_network_list_new(struct loc_ctx* ctx, struct loc_network_list** list);
5342 -struct loc_network_list* loc_network_list_ref(struct loc_network_list* list);
5343 -struct loc_network_list* loc_network_list_unref(struct loc_network_list* list);
5344 -size_t loc_network_list_size(struct loc_network_list* list);
5345 -int loc_network_list_empty(struct loc_network_list* list);
5346 -void loc_network_list_clear(struct loc_network_list* list);
5347 -void loc_network_list_dump(struct loc_network_list* list);
5348 -struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index);
5349 -int loc_network_list_push(struct loc_network_list* list, struct loc_network* network);
5350 -struct loc_network* loc_network_list_pop(struct loc_network_list* list);
5351 -struct loc_network* loc_network_list_pop_first(struct loc_network_list* list);
5352 -int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network);
5353 -void loc_network_list_sort(struct loc_network_list* list);
5354 -void loc_network_list_reverse(struct loc_network_list* list);
5355 -int loc_network_list_merge(struct loc_network_list* self, struct loc_network_list* other);
5356 -
5357 #ifdef LIBLOC_PRIVATE
5358
5359 int loc_network_to_database_v1(struct loc_network* network, struct loc_database_network_v1* dbobj);
5360 diff --git a/src/network-list.c b/src/network-list.c
5361 new file mode 100644
5362 index 0000000..1f6e80e
5363 --- /dev/null
5364 +++ b/src/network-list.c
5365 @@ -0,0 +1,224 @@
5366 +/*
5367 + libloc - A library to determine the location of someone on the Internet
5368 +
5369 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
5370 +
5371 + This library is free software; you can redistribute it and/or
5372 + modify it under the terms of the GNU Lesser General Public
5373 + License as published by the Free Software Foundation; either
5374 + version 2.1 of the License, or (at your option) any later version.
5375 +
5376 + This library is distributed in the hope that it will be useful,
5377 + but WITHOUT ANY WARRANTY; without even the implied warranty of
5378 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
5379 + Lesser General Public License for more details.
5380 +*/
5381 +
5382 +#include <errno.h>
5383 +#include <stdlib.h>
5384 +
5385 +#include <loc/libloc.h>
5386 +#include <loc/network.h>
5387 +#include <loc/private.h>
5388 +
5389 +struct loc_network_list {
5390 + struct loc_ctx* ctx;
5391 + int refcount;
5392 +
5393 + struct loc_network* list[1024];
5394 + size_t size;
5395 + size_t max_size;
5396 +};
5397 +
5398 +LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
5399 + struct loc_network_list** list) {
5400 + struct loc_network_list* l = calloc(1, sizeof(*l));
5401 + if (!l)
5402 + return -ENOMEM;
5403 +
5404 + l->ctx = loc_ref(ctx);
5405 + l->refcount = 1;
5406 +
5407 + // Do not allow this list to grow larger than this
5408 + l->max_size = 1024;
5409 +
5410 + DEBUG(l->ctx, "Network list allocated at %p\n", l);
5411 + *list = l;
5412 + return 0;
5413 +}
5414 +
5415 +LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
5416 + list->refcount++;
5417 +
5418 + return list;
5419 +}
5420 +
5421 +static void loc_network_list_free(struct loc_network_list* list) {
5422 + DEBUG(list->ctx, "Releasing network list at %p\n", list);
5423 +
5424 + for (unsigned int i = 0; i < list->size; i++)
5425 + loc_network_unref(list->list[i]);
5426 +
5427 + loc_unref(list->ctx);
5428 + free(list);
5429 +}
5430 +
5431 +LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
5432 + if (!list)
5433 + return NULL;
5434 +
5435 + if (--list->refcount > 0)
5436 + return list;
5437 +
5438 + loc_network_list_free(list);
5439 + return NULL;
5440 +}
5441 +
5442 +LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
5443 + return list->size;
5444 +}
5445 +
5446 +LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
5447 + return list->size == 0;
5448 +}
5449 +
5450 +LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
5451 + for (unsigned int i = 0; i < list->size; i++)
5452 + loc_network_unref(list->list[i]);
5453 +
5454 + list->size = 0;
5455 +}
5456 +
5457 +LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
5458 + struct loc_network* network;
5459 + char* s;
5460 +
5461 + for (unsigned int i = 0; i < list->size; i++) {
5462 + network = list->list[i];
5463 +
5464 + s = loc_network_str(network);
5465 +
5466 + INFO(list->ctx, "%s\n", s);
5467 + free(s);
5468 + }
5469 +}
5470 +
5471 +LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
5472 + // Check index
5473 + if (index >= list->size)
5474 + return NULL;
5475 +
5476 + return loc_network_ref(list->list[index]);
5477 +}
5478 +
5479 +LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
5480 + // Do not add networks that are already on the list
5481 + if (loc_network_list_contains(list, network))
5482 + return 0;
5483 +
5484 + // Check if we have space left
5485 + if (list->size == list->max_size) {
5486 + ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
5487 + return -ENOMEM;
5488 + }
5489 +
5490 + DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
5491 +
5492 + list->list[list->size++] = loc_network_ref(network);
5493 +
5494 + return 0;
5495 +}
5496 +
5497 +LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
5498 + // Return nothing when empty
5499 + if (loc_network_list_empty(list)) {
5500 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
5501 + return NULL;
5502 + }
5503 +
5504 + struct loc_network* network = list->list[--list->size];
5505 +
5506 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
5507 +
5508 + return network;
5509 +}
5510 +
5511 +LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
5512 + // Return nothing when empty
5513 + if (loc_network_list_empty(list)) {
5514 + DEBUG(list->ctx, "%p: Popped empty stack\n", list);
5515 + return NULL;
5516 + }
5517 +
5518 + struct loc_network* network = list->list[0];
5519 +
5520 + // Move all elements to the top of the stack
5521 + for (unsigned int i = 0; i < --list->size; i++) {
5522 + list->list[i] = list->list[i+1];
5523 + }
5524 +
5525 + DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
5526 +
5527 + return network;
5528 +}
5529 +
5530 +LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
5531 + for (unsigned int i = 0; i < list->size; i++) {
5532 + if (loc_network_eq(list->list[i], network))
5533 + return 1;
5534 + }
5535 +
5536 + return 0;
5537 +}
5538 +
5539 +static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
5540 + // Do nothing for invalid indices
5541 + if (i1 >= list->size || i2 >= list->size)
5542 + return;
5543 +
5544 + struct loc_network* network1 = list->list[i1];
5545 + struct loc_network* network2 = list->list[i2];
5546 +
5547 + list->list[i1] = network2;
5548 + list->list[i2] = network1;
5549 +}
5550 +
5551 +LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
5552 + unsigned int i = 0;
5553 + unsigned int j = list->size - 1;
5554 +
5555 + while (i < j) {
5556 + loc_network_list_swap(list, i++, j--);
5557 + }
5558 +}
5559 +
5560 +LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
5561 + unsigned int n = list->size;
5562 + int swapped;
5563 +
5564 + do {
5565 + swapped = 0;
5566 +
5567 + for (unsigned int i = 1; i < n; i++) {
5568 + if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
5569 + loc_network_list_swap(list, i-1, i);
5570 + swapped = 1;
5571 + }
5572 + }
5573 +
5574 + n--;
5575 + } while (swapped);
5576 +}
5577 +
5578 +LOC_EXPORT int loc_network_list_merge(
5579 + struct loc_network_list* self, struct loc_network_list* other) {
5580 + int r;
5581 +
5582 + for (unsigned int i = 0; i < other->size; i++) {
5583 + r = loc_network_list_push(self, other->list[i]);
5584 + if (r)
5585 + return r;
5586 + }
5587 +
5588 + return 0;
5589 +}
5590 diff --git a/src/network.c b/src/network.c
5591 index d015579..28ca2df 100644
5592 --- a/src/network.c
5593 +++ b/src/network.c
5594 @@ -29,6 +29,7 @@
5595 #include <loc/compat.h>
5596 #include <loc/country.h>
5597 #include <loc/network.h>
5598 +#include <loc/network-list.h>
5599 #include <loc/private.h>
5600
5601 struct loc_network {
5602 @@ -436,7 +437,7 @@ LOC_EXPORT int loc_network_eq(struct loc_network* self, struct loc_network* othe
5603 return 1;
5604 }
5605
5606 -static int loc_network_gt(struct loc_network* self, struct loc_network* other) {
5607 +LOC_EXPORT int loc_network_gt(struct loc_network* self, struct loc_network* other) {
5608 // Families must match
5609 if (self->family != other->family)
5610 return -1;
5611 @@ -1106,207 +1107,3 @@ int loc_network_tree_node_is_leaf(struct loc_network_tree_node* node) {
5612 struct loc_network* loc_network_tree_node_get_network(struct loc_network_tree_node* node) {
5613 return loc_network_ref(node->network);
5614 }
5615 -
5616 -// List
5617 -
5618 -struct loc_network_list {
5619 - struct loc_ctx* ctx;
5620 - int refcount;
5621 -
5622 - struct loc_network* list[1024];
5623 - size_t size;
5624 - size_t max_size;
5625 -};
5626 -
5627 -LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
5628 - struct loc_network_list** list) {
5629 - struct loc_network_list* l = calloc(1, sizeof(*l));
5630 - if (!l)
5631 - return -ENOMEM;
5632 -
5633 - l->ctx = loc_ref(ctx);
5634 - l->refcount = 1;
5635 -
5636 - // Do not allow this list to grow larger than this
5637 - l->max_size = 1024;
5638 -
5639 - DEBUG(l->ctx, "Network list allocated at %p\n", l);
5640 - *list = l;
5641 - return 0;
5642 -}
5643 -
5644 -LOC_EXPORT struct loc_network_list* loc_network_list_ref(struct loc_network_list* list) {
5645 - list->refcount++;
5646 -
5647 - return list;
5648 -}
5649 -
5650 -static void loc_network_list_free(struct loc_network_list* list) {
5651 - DEBUG(list->ctx, "Releasing network list at %p\n", list);
5652 -
5653 - for (unsigned int i = 0; i < list->size; i++)
5654 - loc_network_unref(list->list[i]);
5655 -
5656 - loc_unref(list->ctx);
5657 - free(list);
5658 -}
5659 -
5660 -LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
5661 - if (!list)
5662 - return NULL;
5663 -
5664 - if (--list->refcount > 0)
5665 - return list;
5666 -
5667 - loc_network_list_free(list);
5668 - return NULL;
5669 -}
5670 -
5671 -LOC_EXPORT size_t loc_network_list_size(struct loc_network_list* list) {
5672 - return list->size;
5673 -}
5674 -
5675 -LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
5676 - return list->size == 0;
5677 -}
5678 -
5679 -LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
5680 - for (unsigned int i = 0; i < list->size; i++)
5681 - loc_network_unref(list->list[i]);
5682 -
5683 - list->size = 0;
5684 -}
5685 -
5686 -LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
5687 - struct loc_network* network;
5688 - char* s;
5689 -
5690 - for (unsigned int i = 0; i < list->size; i++) {
5691 - network = list->list[i];
5692 -
5693 - s = loc_network_str(network);
5694 -
5695 - INFO(list->ctx, "%s\n", s);
5696 - free(s);
5697 - }
5698 -}
5699 -
5700 -LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* list, size_t index) {
5701 - // Check index
5702 - if (index >= list->size)
5703 - return NULL;
5704 -
5705 - return loc_network_ref(list->list[index]);
5706 -}
5707 -
5708 -LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
5709 - // Do not add networks that are already on the list
5710 - if (loc_network_list_contains(list, network))
5711 - return 0;
5712 -
5713 - // Check if we have space left
5714 - if (list->size == list->max_size) {
5715 - ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
5716 - return -ENOMEM;
5717 - }
5718 -
5719 - DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
5720 -
5721 - list->list[list->size++] = loc_network_ref(network);
5722 -
5723 - return 0;
5724 -}
5725 -
5726 -LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* list) {
5727 - // Return nothing when empty
5728 - if (loc_network_list_empty(list)) {
5729 - DEBUG(list->ctx, "%p: Popped empty stack\n", list);
5730 - return NULL;
5731 - }
5732 -
5733 - struct loc_network* network = list->list[--list->size];
5734 -
5735 - DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
5736 -
5737 - return network;
5738 -}
5739 -
5740 -LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_list* list) {
5741 - // Return nothing when empty
5742 - if (loc_network_list_empty(list)) {
5743 - DEBUG(list->ctx, "%p: Popped empty stack\n", list);
5744 - return NULL;
5745 - }
5746 -
5747 - struct loc_network* network = list->list[0];
5748 -
5749 - // Move all elements to the top of the stack
5750 - for (unsigned int i = 0; i < --list->size; i++) {
5751 - list->list[i] = list->list[i+1];
5752 - }
5753 -
5754 - DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
5755 -
5756 - return network;
5757 -}
5758 -
5759 -LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
5760 - for (unsigned int i = 0; i < list->size; i++) {
5761 - if (loc_network_eq(list->list[i], network))
5762 - return 1;
5763 - }
5764 -
5765 - return 0;
5766 -}
5767 -
5768 -static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1, unsigned int i2) {
5769 - // Do nothing for invalid indices
5770 - if (i1 >= list->size || i2 >= list->size)
5771 - return;
5772 -
5773 - struct loc_network* network1 = list->list[i1];
5774 - struct loc_network* network2 = list->list[i2];
5775 -
5776 - list->list[i1] = network2;
5777 - list->list[i2] = network1;
5778 -}
5779 -
5780 -LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
5781 - unsigned int i = 0;
5782 - unsigned int j = list->size - 1;
5783 -
5784 - while (i < j) {
5785 - loc_network_list_swap(list, i++, j--);
5786 - }
5787 -}
5788 -
5789 -LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
5790 - unsigned int n = list->size;
5791 - int swapped;
5792 -
5793 - do {
5794 - swapped = 0;
5795 -
5796 - for (unsigned int i = 1; i < n; i++) {
5797 - if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
5798 - loc_network_list_swap(list, i-1, i);
5799 - swapped = 1;
5800 - }
5801 - }
5802 -
5803 - n--;
5804 - } while (swapped);
5805 -}
5806 -
5807 -LOC_EXPORT int loc_network_list_merge(
5808 - struct loc_network_list* self, struct loc_network_list* other) {
5809 - int r;
5810 -
5811 - for (unsigned int i = 0; i < other->size; i++) {
5812 - r = loc_network_list_push(self, other->list[i]);
5813 - if (r)
5814 - return r;
5815 - }
5816 -
5817 - return 0;
5818 -}
5819 diff --git a/src/python/network.c b/src/python/network.c
5820 index 11f672b..ed91d65 100644
5821 --- a/src/python/network.c
5822 +++ b/src/python/network.c
5823 @@ -20,6 +20,7 @@
5824
5825 #include <loc/libloc.h>
5826 #include <loc/network.h>
5827 +#include <loc/network-list.h>
5828
5829 #include "locationmodule.h"
5830 #include "network.h"
5831 --
5832 2.20.1
5833
5834 From e646a8f35ec7eff009414b3fd107c9af5cf39a86 Mon Sep 17 00:00:00 2001
5835 From: Michael Tremer <michael.tremer@ipfire.org>
5836 Date: Mon, 16 Nov 2020 15:13:28 +0000
5837 Subject: [PATCH 56/70] Implement filtering for multiple countries in the
5838 enumerator
5839
5840 This will allow us to speed up the export of the database
5841 if only a few countries should be returned.
5842
5843 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
5844 ---
5845 Makefile.am | 2 +
5846 src/country-list.c | 138 +++++++++++++++++++++++++++++++++++++++++
5847 src/country.c | 3 +
5848 src/database.c | 47 ++++++--------
5849 src/libloc.sym | 15 ++++-
5850 src/loc/country-list.h | 43 +++++++++++++
5851 src/loc/database.h | 5 +-
5852 src/python/database.c | 57 ++++++++++++++---
5853 8 files changed, 274 insertions(+), 36 deletions(-)
5854 create mode 100644 src/country-list.c
5855 create mode 100644 src/loc/country-list.h
5856
5857 diff --git a/Makefile.am b/Makefile.am
5858 index f0d8c4c..f4ca3c8 100644
5859 --- a/Makefile.am
5860 +++ b/Makefile.am
5861 @@ -93,6 +93,7 @@ pkginclude_HEADERS = \
5862 src/loc/as.h \
5863 src/loc/compat.h \
5864 src/loc/country.h \
5865 + src/loc/country-list.h \
5866 src/loc/database.h \
5867 src/loc/format.h \
5868 src/loc/network.h \
5869 @@ -109,6 +110,7 @@ src_libloc_la_SOURCES = \
5870 src/libloc.c \
5871 src/as.c \
5872 src/country.c \
5873 + src/country-list.c \
5874 src/database.c \
5875 src/network.c \
5876 src/network-list.c \
5877 diff --git a/src/country-list.c b/src/country-list.c
5878 new file mode 100644
5879 index 0000000..ae0d71a
5880 --- /dev/null
5881 +++ b/src/country-list.c
5882 @@ -0,0 +1,138 @@
5883 +/*
5884 + libloc - A library to determine the location of someone on the Internet
5885 +
5886 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
5887 +
5888 + This library is free software; you can redistribute it and/or
5889 + modify it under the terms of the GNU Lesser General Public
5890 + License as published by the Free Software Foundation; either
5891 + version 2.1 of the License, or (at your option) any later version.
5892 +
5893 + This library is distributed in the hope that it will be useful,
5894 + but WITHOUT ANY WARRANTY; without even the implied warranty of
5895 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
5896 + Lesser General Public License for more details.
5897 +*/
5898 +
5899 +#include <errno.h>
5900 +#include <stdlib.h>
5901 +
5902 +#include <loc/country.h>
5903 +#include <loc/country-list.h>
5904 +#include <loc/private.h>
5905 +
5906 +struct loc_country_list {
5907 + struct loc_ctx* ctx;
5908 + int refcount;
5909 +
5910 + struct loc_country* list[1024];
5911 + size_t size;
5912 + size_t max_size;
5913 +};
5914 +
5915 +LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
5916 + struct loc_country_list** list) {
5917 + struct loc_country_list* l = calloc(1, sizeof(*l));
5918 + if (!l)
5919 + return -ENOMEM;
5920 +
5921 + l->ctx = loc_ref(ctx);
5922 + l->refcount = 1;
5923 +
5924 + // Do not allow this list to grow larger than this
5925 + l->max_size = 1024;
5926 +
5927 + DEBUG(l->ctx, "Country list allocated at %p\n", l);
5928 + *list = l;
5929 +
5930 + return 0;
5931 +}
5932 +
5933 +LOC_EXPORT struct loc_country_list* loc_country_list_ref(struct loc_country_list* list) {
5934 + list->refcount++;
5935 +
5936 + return list;
5937 +}
5938 +
5939 +static void loc_country_list_free(struct loc_country_list* list) {
5940 + DEBUG(list->ctx, "Releasing country list at %p\n", list);
5941 +
5942 + loc_country_list_clear(list);
5943 +
5944 + loc_unref(list->ctx);
5945 + free(list);
5946 +}
5947 +
5948 +LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) {
5949 + if (!list)
5950 + return NULL;
5951 +
5952 + if (--list->refcount > 0)
5953 + return list;
5954 +
5955 + loc_country_list_free(list);
5956 + return NULL;
5957 +}
5958 +
5959 +LOC_EXPORT size_t loc_country_list_size(struct loc_country_list* list) {
5960 + return list->size;
5961 +}
5962 +
5963 +LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
5964 + return list->size == 0;
5965 +}
5966 +
5967 +LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
5968 + for (unsigned int i = 0; i < list->size; i++)
5969 + loc_country_unref(list->list[i]);
5970 +}
5971 +
5972 +LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
5973 + // Check index
5974 + if (index >= list->size)
5975 + return NULL;
5976 +
5977 + return loc_country_ref(list->list[index]);
5978 +}
5979 +
5980 +LOC_EXPORT int loc_country_list_append(
5981 + struct loc_country_list* list, struct loc_country* country) {
5982 + if (loc_country_list_contains(list, country))
5983 + return 0;
5984 +
5985 + // Check if we have space left
5986 + if (list->size == list->max_size) {
5987 + ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list);
5988 + return -ENOMEM;
5989 + }
5990 +
5991 + DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
5992 +
5993 + list->list[list->size++] = loc_country_ref(country);
5994 +
5995 + return 0;
5996 +}
5997 +
5998 +LOC_EXPORT int loc_country_list_contains(
5999 + struct loc_country_list* list, struct loc_country* country) {
6000 + for (unsigned int i = 0; i < list->size; i++) {
6001 + if (loc_country_cmp(country, list->list[i]) == 0)
6002 + return 1;
6003 + }
6004 +
6005 + return 0;
6006 +}
6007 +
6008 +LOC_EXPORT int loc_country_list_contains_code(
6009 + struct loc_country_list* list, const char* code) {
6010 + struct loc_country* country;
6011 +
6012 + int r = loc_country_new(list->ctx, &country, code);
6013 + if (r)
6014 + return -1;
6015 +
6016 + r = loc_country_list_contains(list, country);
6017 + loc_country_unref(country);
6018 +
6019 + return r;
6020 +}
6021 diff --git a/src/country.c b/src/country.c
6022 index 2ba93e6..7aac0db 100644
6023 --- a/src/country.c
6024 +++ b/src/country.c
6025 @@ -34,6 +34,9 @@ struct loc_country {
6026 };
6027
6028 LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country, const char* country_code) {
6029 + if (!loc_country_code_is_valid(country_code))
6030 + return -EINVAL;
6031 +
6032 struct loc_country* c = calloc(1, sizeof(*c));
6033 if (!c)
6034 return -ENOMEM;
6035 diff --git a/src/database.c b/src/database.c
6036 index b9d870f..29823b2 100644
6037 --- a/src/database.c
6038 +++ b/src/database.c
6039 @@ -40,6 +40,7 @@
6040 #include <loc/as.h>
6041 #include <loc/compat.h>
6042 #include <loc/country.h>
6043 +#include <loc/country-list.h>
6044 #include <loc/database.h>
6045 #include <loc/format.h>
6046 #include <loc/network.h>
6047 @@ -99,7 +100,7 @@ struct loc_database_enumerator {
6048
6049 // Search string
6050 char* string;
6051 - char country_code[3];
6052 + struct loc_country_list* countries;
6053 uint32_t asn;
6054 enum loc_network_flags flags;
6055 int family;
6056 @@ -1017,33 +1018,20 @@ LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator
6057 return 0;
6058 }
6059
6060 -LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) {
6061 - // Set empty country code
6062 - if (!country_code || !*country_code) {
6063 - *enumerator->country_code = '\0';
6064 - return 0;
6065 - }
6066 +LOC_EXPORT struct loc_country_list* loc_database_enumerator_get_countries(
6067 + struct loc_database_enumerator* enumerator) {
6068 + if (!enumerator->countries)
6069 + return NULL;
6070
6071 - // Treat A1, A2, A3 as special country codes,
6072 - // but perform search for flags instead
6073 - if (strcmp(country_code, "A1") == 0) {
6074 - return loc_database_enumerator_set_flag(enumerator,
6075 - LOC_NETWORK_FLAG_ANONYMOUS_PROXY);
6076 - } else if (strcmp(country_code, "A2") == 0) {
6077 - return loc_database_enumerator_set_flag(enumerator,
6078 - LOC_NETWORK_FLAG_SATELLITE_PROVIDER);
6079 - } else if (strcmp(country_code, "A3") == 0) {
6080 - return loc_database_enumerator_set_flag(enumerator,
6081 - LOC_NETWORK_FLAG_ANYCAST);
6082 - }
6083 + return loc_country_list_ref(enumerator->countries);
6084 +}
6085
6086 - // Country codes must be two characters
6087 - if (!loc_country_code_is_valid(country_code))
6088 - return -EINVAL;
6089 +LOC_EXPORT int loc_database_enumerator_set_countries(
6090 + struct loc_database_enumerator* enumerator, struct loc_country_list* countries) {
6091 + if (enumerator->countries)
6092 + loc_country_list_unref(enumerator->countries);
6093
6094 - for (unsigned int i = 0; i < 3; i++) {
6095 - enumerator->country_code[i] = country_code[i];
6096 - }
6097 + enumerator->countries = loc_country_list_ref(countries);
6098
6099 return 0;
6100 }
6101 @@ -1129,6 +1117,12 @@ static int loc_database_enumerator_stack_push_node(
6102 return 0;
6103 }
6104
6105 +static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) {
6106 + const char* country_code = loc_network_get_country_code(network);
6107 +
6108 + return loc_country_list_contains_code(countries, country_code);
6109 +}
6110 +
6111 static int loc_database_enumerator_filter_network(
6112 struct loc_database_enumerator* enumerator, struct loc_network* network) {
6113 // Skip if the family does not match
6114 @@ -1136,8 +1130,7 @@ static int loc_database_enumerator_filter_network(
6115 return 1;
6116
6117 // Skip if the country code does not match
6118 - if (*enumerator->country_code &&
6119 - !loc_network_match_country_code(network, enumerator->country_code))
6120 + if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries))
6121 return 1;
6122
6123 // Skip if the ASN does not match
6124 diff --git a/src/libloc.sym b/src/libloc.sym
6125 index 453a1be..40e9f88 100644
6126 --- a/src/libloc.sym
6127 +++ b/src/libloc.sym
6128 @@ -49,6 +49,18 @@ global:
6129 loc_country_set_name;
6130 loc_country_unref;
6131
6132 + # Country List
6133 + loc_country_list_append;
6134 + loc_country_list_clear;
6135 + loc_country_list_contains;
6136 + loc_country_list_contains_code;
6137 + loc_country_list_empty;
6138 + loc_country_list_get;
6139 + loc_country_list_new;
6140 + loc_country_list_ref;
6141 + loc_country_list_size;
6142 + loc_country_list_unref;
6143 +
6144 # Database
6145 loc_database_add_as;
6146 loc_database_count_as;
6147 @@ -66,13 +78,14 @@ global:
6148 loc_database_verify;
6149
6150 # Database Enumerator
6151 + loc_database_enumerator_get_countries;
6152 loc_database_enumerator_new;
6153 loc_database_enumerator_next_as;
6154 loc_database_enumerator_next_country;
6155 loc_database_enumerator_next_network;
6156 loc_database_enumerator_ref;
6157 loc_database_enumerator_set_asn;
6158 - loc_database_enumerator_set_country_code;
6159 + loc_database_enumerator_set_countries;
6160 loc_database_enumerator_set_family;
6161 loc_database_enumerator_set_flag;
6162 loc_database_enumerator_set_string;
6163 diff --git a/src/loc/country-list.h b/src/loc/country-list.h
6164 new file mode 100644
6165 index 0000000..a7f818a
6166 --- /dev/null
6167 +++ b/src/loc/country-list.h
6168 @@ -0,0 +1,43 @@
6169 +/*
6170 + libloc - A library to determine the location of someone on the Internet
6171 +
6172 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
6173 +
6174 + This library is free software; you can redistribute it and/or
6175 + modify it under the terms of the GNU Lesser General Public
6176 + License as published by the Free Software Foundation; either
6177 + version 2.1 of the License, or (at your option) any later version.
6178 +
6179 + This library is distributed in the hope that it will be useful,
6180 + but WITHOUT ANY WARRANTY; without even the implied warranty of
6181 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
6182 + Lesser General Public License for more details.
6183 +*/
6184 +
6185 +#ifndef LIBLOC_COUNTRY_LIST_H
6186 +#define LIBLOC_COUNTRY_LIST_H
6187 +
6188 +#include <stdlib.h>
6189 +
6190 +#include <loc/libloc.h>
6191 +#include <loc/country.h>
6192 +
6193 +struct loc_country_list;
6194 +
6195 +int loc_country_list_new(struct loc_ctx* ctx, struct loc_country_list** list);
6196 +struct loc_country_list* loc_country_list_ref(struct loc_country_list* list);
6197 +struct loc_country_list* loc_country_list_unref(struct loc_country_list* list);
6198 +
6199 +size_t loc_country_list_size(struct loc_country_list* list);
6200 +int loc_country_list_empty(struct loc_country_list* list);
6201 +void loc_country_list_clear(struct loc_country_list* list);
6202 +
6203 +struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index);
6204 +int loc_country_list_append(struct loc_country_list* list, struct loc_country* country);
6205 +
6206 +int loc_country_list_contains(
6207 + struct loc_country_list* list, struct loc_country* country);
6208 +int loc_country_list_contains_code(
6209 + struct loc_country_list* list, const char* code);
6210 +
6211 +#endif
6212 diff --git a/src/loc/database.h b/src/loc/database.h
6213 index 14eb5ea..246e5c5 100644
6214 --- a/src/loc/database.h
6215 +++ b/src/loc/database.h
6216 @@ -25,6 +25,7 @@
6217 #include <loc/network.h>
6218 #include <loc/as.h>
6219 #include <loc/country.h>
6220 +#include <loc/country-list.h>
6221
6222 struct loc_database;
6223 int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f);
6224 @@ -66,7 +67,9 @@ struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_
6225 struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator);
6226
6227 int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string);
6228 -int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code);
6229 +struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
6230 +int loc_database_enumerator_set_countries(
6231 + struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
6232 int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
6233 int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
6234 int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
6235 diff --git a/src/python/database.c b/src/python/database.c
6236 index d169547..e6f6f37 100644
6237 --- a/src/python/database.c
6238 +++ b/src/python/database.c
6239 @@ -258,14 +258,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
6240 }
6241
6242 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
6243 - char* kwlist[] = { "country_code", "asn", "flags", "family", "flatten", NULL };
6244 - const char* country_code = NULL;
6245 + char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL };
6246 + PyObject* country_codes = NULL;
6247 unsigned int asn = 0;
6248 int flags = 0;
6249 int family = 0;
6250 int flatten = 0;
6251
6252 - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|siiip", kwlist, &country_code, &asn, &flags, &family, &flatten))
6253 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist,
6254 + &PyList_Type, &country_codes, &asn, &flags, &family, &flatten))
6255 return NULL;
6256
6257 struct loc_database_enumerator* enumerator;
6258 @@ -277,13 +278,55 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
6259 }
6260
6261 // Set country code we are searching for
6262 - if (country_code) {
6263 - r = loc_database_enumerator_set_country_code(enumerator, country_code);
6264 -
6265 + if (country_codes) {
6266 + struct loc_country_list* countries;
6267 + r = loc_country_list_new(loc_ctx, &countries);
6268 if (r) {
6269 - PyErr_SetFromErrno(PyExc_SystemError);
6270 + PyErr_SetString(PyExc_SystemError, "Could not create country list");
6271 return NULL;
6272 }
6273 +
6274 + for (unsigned int i = 0; i < PyList_Size(country_codes); i++) {
6275 + PyObject* item = PyList_GetItem(country_codes, i);
6276 +
6277 + if (!PyUnicode_Check(item)) {
6278 + PyErr_SetString(PyExc_TypeError, "Country codes must be strings");
6279 + loc_country_list_unref(countries);
6280 + return NULL;
6281 + }
6282 +
6283 + const char* country_code = PyUnicode_AsUTF8(item);
6284 +
6285 + struct loc_country* country;
6286 + r = loc_country_new(loc_ctx, &country, country_code);
6287 + if (r) {
6288 + if (r == -EINVAL) {
6289 + PyErr_Format(PyExc_ValueError, "Invalid country code: %s", country_code);
6290 + } else {
6291 + PyErr_SetString(PyExc_SystemError, "Could not create country");
6292 + }
6293 +
6294 + loc_country_list_unref(countries);
6295 + return NULL;
6296 + }
6297 +
6298 + // Append it to the list
6299 + r = loc_country_list_append(countries, country);
6300 + if (r) {
6301 + PyErr_SetString(PyExc_SystemError, "Could not append country to the list");
6302 +
6303 + loc_country_list_unref(countries);
6304 + loc_country_unref(country);
6305 + return NULL;
6306 + }
6307 +
6308 + loc_country_unref(country);
6309 + }
6310 +
6311 + loc_database_enumerator_set_countries(enumerator, countries);
6312 +
6313 + Py_DECREF(country_codes);
6314 + loc_country_list_unref(countries);
6315 }
6316
6317 // Set the ASN we are searching for
6318 --
6319 2.20.1
6320
6321 From 7af51f8a579c79714992a3e175036fb511139310 Mon Sep 17 00:00:00 2001
6322 From: Michael Tremer <michael.tremer@ipfire.org>
6323 Date: Mon, 16 Nov 2020 15:20:50 +0000
6324 Subject: [PATCH 57/70] python: Only return country codes we want
6325
6326 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6327 ---
6328 src/python/export.py | 8 +++++++-
6329 1 file changed, 7 insertions(+), 1 deletion(-)
6330
6331 diff --git a/src/python/export.py b/src/python/export.py
6332 index be4a68e..5e7fe53 100644
6333 --- a/src/python/export.py
6334 +++ b/src/python/export.py
6335 @@ -184,8 +184,14 @@ class Exporter(object):
6336
6337 writers[asn] = self.writer.open(self.db, filename, prefix="AS%s" % asn)
6338
6339 + # Filter countries from special country codes
6340 + country_codes = [
6341 + country_code for country_code in countries if not country_code in flags.values()
6342 + ]
6343 +
6344 # Get all networks that match the family
6345 - networks = self.db.search_networks(family=family, flatten=True)
6346 + networks = self.db.search_networks(family=family,
6347 + country_codes=country_codes, flatten=True)
6348
6349 # Walk through all networks
6350 for network in networks:
6351 --
6352 2.20.1
6353
6354 From bd1dc6bf6fe4ce40bf12e7426e283b31afd274e1 Mon Sep 17 00:00:00 2001
6355 From: Michael Tremer <michael.tremer@ipfire.org>
6356 Date: Mon, 16 Nov 2020 15:25:15 +0000
6357 Subject: [PATCH 58/70] database: Filter flags in C
6358
6359 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6360 ---
6361 src/python/export.py | 16 +++++++++++-----
6362 1 file changed, 11 insertions(+), 5 deletions(-)
6363
6364 diff --git a/src/python/export.py b/src/python/export.py
6365 index 5e7fe53..739742f 100644
6366 --- a/src/python/export.py
6367 +++ b/src/python/export.py
6368 @@ -29,7 +29,7 @@ import _location
6369 log = logging.getLogger("location.export")
6370 log.propagate = 1
6371
6372 -flags = {
6373 +FLAGS = {
6374 _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
6375 _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
6376 _location.NETWORK_FLAG_ANYCAST : "A3",
6377 @@ -186,12 +186,18 @@ class Exporter(object):
6378
6379 # Filter countries from special country codes
6380 country_codes = [
6381 - country_code for country_code in countries if not country_code in flags.values()
6382 + country_code for country_code in countries if not country_code in FLAGS.values()
6383 ]
6384
6385 + # Collect flags
6386 + flags = 0
6387 + for flag in FLAGS:
6388 + if FLAGS[flag] in countries:
6389 + flags |= flag
6390 +
6391 # Get all networks that match the family
6392 networks = self.db.search_networks(family=family,
6393 - country_codes=country_codes, flatten=True)
6394 + country_codes=country_codes, flags=flags, flatten=True)
6395
6396 # Walk through all networks
6397 for network in networks:
6398 @@ -208,10 +214,10 @@ class Exporter(object):
6399 pass
6400
6401 # Handle flags
6402 - for flag in flags:
6403 + for flag in FLAGS:
6404 if network.has_flag(flag):
6405 # Fetch the "fake" country code
6406 - country = flags[flag]
6407 + country = FLAGS[flag]
6408
6409 try:
6410 writers[country].write(network)
6411 --
6412 2.20.1
6413
6414 From 84a2f0c2d9cbf8ae4225802c29ccba86561c77ed Mon Sep 17 00:00:00 2001
6415 From: Michael Tremer <michael.tremer@ipfire.org>
6416 Date: Tue, 17 Nov 2020 16:46:48 +0000
6417 Subject: [PATCH 59/70] as: Add list for easier processing
6418
6419 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6420 ---
6421 Makefile.am | 2 +
6422 src/as-list.c | 138 ++++++++++++++++++++++++++++++++++++++++++
6423 src/database.c | 29 +++++++--
6424 src/libloc.sym | 15 ++++-
6425 src/loc/as-list.h | 41 +++++++++++++
6426 src/loc/database.h | 5 +-
6427 src/python/database.c | 58 ++++++++++++++++--
6428 src/python/export.py | 2 +-
6429 8 files changed, 275 insertions(+), 15 deletions(-)
6430 create mode 100644 src/as-list.c
6431 create mode 100644 src/loc/as-list.h
6432
6433 diff --git a/Makefile.am b/Makefile.am
6434 index f4ca3c8..d0cc793 100644
6435 --- a/Makefile.am
6436 +++ b/Makefile.am
6437 @@ -91,6 +91,7 @@ EXTRA_DIST += \
6438 pkginclude_HEADERS = \
6439 src/loc/libloc.h \
6440 src/loc/as.h \
6441 + src/loc/as-list.h \
6442 src/loc/compat.h \
6443 src/loc/country.h \
6444 src/loc/country-list.h \
6445 @@ -109,6 +110,7 @@ lib_LTLIBRARIES = \
6446 src_libloc_la_SOURCES = \
6447 src/libloc.c \
6448 src/as.c \
6449 + src/as-list.c \
6450 src/country.c \
6451 src/country-list.c \
6452 src/database.c \
6453 diff --git a/src/as-list.c b/src/as-list.c
6454 new file mode 100644
6455 index 0000000..7c69eb0
6456 --- /dev/null
6457 +++ b/src/as-list.c
6458 @@ -0,0 +1,138 @@
6459 +/*
6460 + libloc - A library to determine the location of someone on the Internet
6461 +
6462 + Copyright (C) 2020 IPFire Development Team <info@ipfire.org>
6463 +
6464 + This library is free software; you can redistribute it and/or
6465 + modify it under the terms of the GNU Lesser General Public
6466 + License as published by the Free Software Foundation; either
6467 + version 2.1 of the License, or (at your option) any later version.
6468 +
6469 + This library is distributed in the hope that it will be useful,
6470 + but WITHOUT ANY WARRANTY; without even the implied warranty of
6471 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
6472 + Lesser General Public License for more details.
6473 +*/
6474 +
6475 +#include <errno.h>
6476 +#include <stdlib.h>
6477 +
6478 +#include <loc/as.h>
6479 +#include <loc/as-list.h>
6480 +#include <loc/private.h>
6481 +
6482 +struct loc_as_list {
6483 + struct loc_ctx* ctx;
6484 + int refcount;
6485 +
6486 + struct loc_as* list[1024];
6487 + size_t size;
6488 + size_t max_size;
6489 +};
6490 +
6491 +LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
6492 + struct loc_as_list** list) {
6493 + struct loc_as_list* l = calloc(1, sizeof(*l));
6494 + if (!l)
6495 + return -ENOMEM;
6496 +
6497 + l->ctx = loc_ref(ctx);
6498 + l->refcount = 1;
6499 +
6500 + // Do not allow this list to grow larger than this
6501 + l->max_size = 1024;
6502 +
6503 + DEBUG(l->ctx, "AS list allocated at %p\n", l);
6504 + *list = l;
6505 +
6506 + return 0;
6507 +}
6508 +
6509 +LOC_EXPORT struct loc_as_list* loc_as_list_ref(struct loc_as_list* list) {
6510 + list->refcount++;
6511 +
6512 + return list;
6513 +}
6514 +
6515 +static void loc_as_list_free(struct loc_as_list* list) {
6516 + DEBUG(list->ctx, "Releasing AS list at %p\n", list);
6517 +
6518 + loc_as_list_clear(list);
6519 +
6520 + loc_unref(list->ctx);
6521 + free(list);
6522 +}
6523 +
6524 +LOC_EXPORT struct loc_as_list* loc_as_list_unref(struct loc_as_list* list) {
6525 + if (!list)
6526 + return NULL;
6527 +
6528 + if (--list->refcount > 0)
6529 + return list;
6530 +
6531 + loc_as_list_free(list);
6532 + return NULL;
6533 +}
6534 +
6535 +LOC_EXPORT size_t loc_as_list_size(struct loc_as_list* list) {
6536 + return list->size;
6537 +}
6538 +
6539 +LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
6540 + return list->size == 0;
6541 +}
6542 +
6543 +LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
6544 + for (unsigned int i = 0; i < list->size; i++)
6545 + loc_as_unref(list->list[i]);
6546 +}
6547 +
6548 +LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
6549 + // Check index
6550 + if (index >= list->size)
6551 + return NULL;
6552 +
6553 + return loc_as_ref(list->list[index]);
6554 +}
6555 +
6556 +LOC_EXPORT int loc_as_list_append(
6557 + struct loc_as_list* list, struct loc_as* as) {
6558 + if (loc_as_list_contains(list, as))
6559 + return 0;
6560 +
6561 + // Check if we have space left
6562 + if (list->size == list->max_size) {
6563 + ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list);
6564 + return -ENOMEM;
6565 + }
6566 +
6567 + DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
6568 +
6569 + list->list[list->size++] = loc_as_ref(as);
6570 +
6571 + return 0;
6572 +}
6573 +
6574 +LOC_EXPORT int loc_as_list_contains(
6575 + struct loc_as_list* list, struct loc_as* as) {
6576 + for (unsigned int i = 0; i < list->size; i++) {
6577 + if (loc_as_cmp(as, list->list[i]) == 0)
6578 + return 1;
6579 + }
6580 +
6581 + return 0;
6582 +}
6583 +
6584 +LOC_EXPORT int loc_as_list_contains_number(
6585 + struct loc_as_list* list, uint32_t number) {
6586 + struct loc_as* as;
6587 +
6588 + int r = loc_as_new(list->ctx, &as, number);
6589 + if (r)
6590 + return -1;
6591 +
6592 + r = loc_as_list_contains(list, as);
6593 + loc_as_unref(as);
6594 +
6595 + return r;
6596 +}
6597 diff --git a/src/database.c b/src/database.c
6598 index 29823b2..51cb5cd 100644
6599 --- a/src/database.c
6600 +++ b/src/database.c
6601 @@ -38,6 +38,7 @@
6602
6603 #include <loc/libloc.h>
6604 #include <loc/as.h>
6605 +#include <loc/as-list.h>
6606 #include <loc/compat.h>
6607 #include <loc/country.h>
6608 #include <loc/country-list.h>
6609 @@ -101,7 +102,7 @@ struct loc_database_enumerator {
6610 // Search string
6611 char* string;
6612 struct loc_country_list* countries;
6613 - uint32_t asn;
6614 + struct loc_as_list* asns;
6615 enum loc_network_flags flags;
6616 int family;
6617
6618 @@ -1036,9 +1037,20 @@ LOC_EXPORT int loc_database_enumerator_set_countries(
6619 return 0;
6620 }
6621
6622 -LOC_EXPORT int loc_database_enumerator_set_asn(
6623 - struct loc_database_enumerator* enumerator, unsigned int asn) {
6624 - enumerator->asn = asn;
6625 +LOC_EXPORT struct loc_as_list* loc_database_enumerator_get_asns(
6626 + struct loc_database_enumerator* enumerator) {
6627 + if (!enumerator->asns)
6628 + return NULL;
6629 +
6630 + return loc_as_list_ref(enumerator->asns);
6631 +}
6632 +
6633 +LOC_EXPORT int loc_database_enumerator_set_asns(
6634 + struct loc_database_enumerator* enumerator, struct loc_as_list* asns) {
6635 + if (enumerator->asns)
6636 + loc_as_list_unref(enumerator->asns);
6637 +
6638 + enumerator->asns = loc_as_list_ref(asns);
6639
6640 return 0;
6641 }
6642 @@ -1123,6 +1135,12 @@ static int loc_network_match_countries(struct loc_network* network, struct loc_c
6643 return loc_country_list_contains_code(countries, country_code);
6644 }
6645
6646 +static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
6647 + uint32_t asn = loc_network_get_asn(network);
6648 +
6649 + return loc_as_list_contains_number(asns, asn);
6650 +}
6651 +
6652 static int loc_database_enumerator_filter_network(
6653 struct loc_database_enumerator* enumerator, struct loc_network* network) {
6654 // Skip if the family does not match
6655 @@ -1134,8 +1152,7 @@ static int loc_database_enumerator_filter_network(
6656 return 1;
6657
6658 // Skip if the ASN does not match
6659 - if (enumerator->asn &&
6660 - !loc_network_match_asn(network, enumerator->asn))
6661 + if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
6662 return 1;
6663
6664 // Skip if flags do not match
6665 diff --git a/src/libloc.sym b/src/libloc.sym
6666 index 40e9f88..53273cd 100644
6667 --- a/src/libloc.sym
6668 +++ b/src/libloc.sym
6669 @@ -37,6 +37,18 @@ global:
6670 loc_as_set_name;
6671 loc_as_unref;
6672
6673 + # AS List
6674 + loc_as_list_append;
6675 + loc_as_list_clear;
6676 + loc_as_list_contains;
6677 + loc_as_list_contains_number;
6678 + loc_as_list_empty;
6679 + loc_as_list_get;
6680 + loc_as_list_new;
6681 + loc_as_list_ref;
6682 + loc_as_list_size;
6683 + loc_as_list_unref;
6684 +
6685 # Country
6686 loc_country_cmp;
6687 loc_country_code_is_valid;
6688 @@ -78,13 +90,14 @@ global:
6689 loc_database_verify;
6690
6691 # Database Enumerator
6692 + loc_database_enumerator_get_asns;
6693 loc_database_enumerator_get_countries;
6694 loc_database_enumerator_new;
6695 loc_database_enumerator_next_as;
6696 loc_database_enumerator_next_country;
6697 loc_database_enumerator_next_network;
6698 loc_database_enumerator_ref;
6699 - loc_database_enumerator_set_asn;
6700 + loc_database_enumerator_set_asns;
6701 loc_database_enumerator_set_countries;
6702 loc_database_enumerator_set_family;
6703 loc_database_enumerator_set_flag;
6704 diff --git a/src/loc/as-list.h b/src/loc/as-list.h
6705 new file mode 100644
6706 index 0000000..7b5c4e8
6707 --- /dev/null
6708 +++ b/src/loc/as-list.h
6709 @@ -0,0 +1,41 @@
6710 +/*
6711 + libloc - A library to determine the location of someone on the Internet
6712 +
6713 + Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
6714 +
6715 + This library is free software; you can redistribute it and/or
6716 + modify it under the terms of the GNU Lesser General Public
6717 + License as published by the Free Software Foundation; either
6718 + version 2.1 of the License, or (at your option) any later version.
6719 +
6720 + This library is distributed in the hope that it will be useful,
6721 + but WITHOUT ANY WARRANTY; without even the implied warranty of
6722 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
6723 + Lesser General Public License for more details.
6724 +*/
6725 +
6726 +#ifndef LIBLOC_AS_LIST_H
6727 +#define LIBLOC_AS_LIST_H
6728 +
6729 +#include <loc/as.h>
6730 +#include <loc/libloc.h>
6731 +
6732 +struct loc_as_list;
6733 +
6734 +int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list);
6735 +struct loc_as_list* loc_as_list_ref(struct loc_as_list* list);
6736 +struct loc_as_list* loc_as_list_unref(struct loc_as_list* list);
6737 +
6738 +size_t loc_as_list_size(struct loc_as_list* list);
6739 +int loc_as_list_empty(struct loc_as_list* list);
6740 +void loc_as_list_clear(struct loc_as_list* list);
6741 +
6742 +struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index);
6743 +int loc_as_list_append(struct loc_as_list* list, struct loc_as* as);
6744 +
6745 +int loc_as_list_contains(
6746 + struct loc_as_list* list, struct loc_as* as);
6747 +int loc_as_list_contains_number(
6748 + struct loc_as_list* list, uint32_t number);
6749 +
6750 +#endif
6751 diff --git a/src/loc/database.h b/src/loc/database.h
6752 index 246e5c5..70801f0 100644
6753 --- a/src/loc/database.h
6754 +++ b/src/loc/database.h
6755 @@ -70,7 +70,10 @@ int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerato
6756 struct loc_country_list* loc_database_enumerator_get_countries(struct loc_database_enumerator* enumerator);
6757 int loc_database_enumerator_set_countries(
6758 struct loc_database_enumerator* enumerator, struct loc_country_list* countries);
6759 -int loc_database_enumerator_set_asn(struct loc_database_enumerator* enumerator, unsigned int asn);
6760 +struct loc_as_list* loc_database_enumerator_get_asns(
6761 + struct loc_database_enumerator* enumerator);
6762 +int loc_database_enumerator_set_asns(
6763 + struct loc_database_enumerator* enumerator, struct loc_as_list* asns);
6764 int loc_database_enumerator_set_flag(struct loc_database_enumerator* enumerator, enum loc_network_flags flag);
6765 int loc_database_enumerator_set_family(struct loc_database_enumerator* enumerator, int family);
6766 int loc_database_enumerator_next_as(
6767 diff --git a/src/python/database.c b/src/python/database.c
6768 index e6f6f37..38a804c 100644
6769 --- a/src/python/database.c
6770 +++ b/src/python/database.c
6771 @@ -17,6 +17,8 @@
6772 #include <Python.h>
6773
6774 #include <loc/libloc.h>
6775 +#include <loc/as.h>
6776 +#include <loc/as-list.h>
6777 #include <loc/database.h>
6778
6779 #include "locationmodule.h"
6780 @@ -258,15 +260,15 @@ static PyObject* Database_networks_flattened(DatabaseObject *self) {
6781 }
6782
6783 static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
6784 - char* kwlist[] = { "country_codes", "asn", "flags", "family", "flatten", NULL };
6785 + char* kwlist[] = { "country_codes", "asns", "flags", "family", "flatten", NULL };
6786 PyObject* country_codes = NULL;
6787 - unsigned int asn = 0;
6788 + PyObject* asn_list = NULL;
6789 int flags = 0;
6790 int family = 0;
6791 int flatten = 0;
6792
6793 - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!iiip", kwlist,
6794 - &PyList_Type, &country_codes, &asn, &flags, &family, &flatten))
6795 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!O!iip", kwlist,
6796 + &PyList_Type, &country_codes, &PyList_Type, &asn_list, &flags, &family, &flatten))
6797 return NULL;
6798
6799 struct loc_database_enumerator* enumerator;
6800 @@ -330,13 +332,57 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
6801 }
6802
6803 // Set the ASN we are searching for
6804 - if (asn) {
6805 - r = loc_database_enumerator_set_asn(enumerator, asn);
6806 + if (asn_list) {
6807 + struct loc_as_list* asns;
6808 + r = loc_as_list_new(loc_ctx, &asns);
6809 + if (r) {
6810 + PyErr_SetString(PyExc_SystemError, "Could not create AS list");
6811 + return NULL;
6812 + }
6813 +
6814 + for (unsigned int i = 0; i < PyList_Size(asn_list); i++) {
6815 + PyObject* item = PyList_GetItem(asn_list, i);
6816 +
6817 + if (!PyLong_Check(item)) {
6818 + PyErr_SetString(PyExc_TypeError, "ASNs must be numbers");
6819 +
6820 + loc_as_list_unref(asns);
6821 + return NULL;
6822 + }
6823 +
6824 + unsigned long number = PyLong_AsLong(item);
6825
6826 + struct loc_as* as;
6827 + r = loc_as_new(loc_ctx, &as, number);
6828 + if (r) {
6829 + PyErr_SetString(PyExc_SystemError, "Could not create AS");
6830 +
6831 + loc_as_list_unref(asns);
6832 + loc_as_unref(as);
6833 + return NULL;
6834 + }
6835 +
6836 + r = loc_as_list_append(asns, as);
6837 + if (r) {
6838 + PyErr_SetString(PyExc_SystemError, "Could not append AS to the list");
6839 +
6840 + loc_as_list_unref(asns);
6841 + loc_as_unref(as);
6842 + return NULL;
6843 + }
6844 +
6845 + loc_as_unref(as);
6846 + }
6847 +
6848 + r = loc_database_enumerator_set_asns(enumerator, asns);
6849 if (r) {
6850 PyErr_SetFromErrno(PyExc_SystemError);
6851 +
6852 + loc_as_list_unref(asns);
6853 return NULL;
6854 }
6855 +
6856 + loc_as_list_unref(asns);
6857 }
6858
6859 // Set the flags we are searching for
6860 diff --git a/src/python/export.py b/src/python/export.py
6861 index 739742f..f675eb3 100644
6862 --- a/src/python/export.py
6863 +++ b/src/python/export.py
6864 @@ -197,7 +197,7 @@ class Exporter(object):
6865
6866 # Get all networks that match the family
6867 networks = self.db.search_networks(family=family,
6868 - country_codes=country_codes, flags=flags, flatten=True)
6869 + country_codes=country_codes, asns=asns, flags=flags, flatten=True)
6870
6871 # Walk through all networks
6872 for network in networks:
6873 --
6874 2.20.1
6875
6876 From 50120b991fc2fa4b7813096de87b42d700faf3e6 Mon Sep 17 00:00:00 2001
6877 From: Michael Tremer <michael.tremer@ipfire.org>
6878 Date: Tue, 17 Nov 2020 16:56:43 +0000
6879 Subject: [PATCH 60/70] database: Simplify network matching code
6880
6881 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6882 ---
6883 src/database.c | 16 ++++++++--------
6884 1 file changed, 8 insertions(+), 8 deletions(-)
6885
6886 diff --git a/src/database.c b/src/database.c
6887 index 51cb5cd..1a354f6 100644
6888 --- a/src/database.c
6889 +++ b/src/database.c
6890 @@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node(
6891 return 0;
6892 }
6893
6894 -static int loc_network_match_countries(struct loc_network* network, struct loc_country_list* countries) {
6895 - const char* country_code = loc_network_get_country_code(network);
6896 -
6897 - return loc_country_list_contains_code(countries, country_code);
6898 -}
6899 -
6900 static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
6901 uint32_t asn = loc_network_get_asn(network);
6902
6903 @@ -1148,8 +1142,14 @@ static int loc_database_enumerator_filter_network(
6904 return 1;
6905
6906 // Skip if the country code does not match
6907 - if (enumerator->countries && !loc_network_match_countries(network, enumerator->countries))
6908 - return 1;
6909 + if (enumerator->countries) {
6910 + if (!loc_country_list_empty(enumerator->countries)) {
6911 + const char* country_code = loc_network_get_country_code(network);
6912 +
6913 + if (!loc_country_list_contains_code(enumerator->countries, country_code))
6914 + return 1;
6915 + }
6916 + }
6917
6918 // Skip if the ASN does not match
6919 if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
6920 --
6921 2.20.1
6922
6923 From c1a36c943181da5cd2aef589a972d5027e529eb8 Mon Sep 17 00:00:00 2001
6924 From: Michael Tremer <michael.tremer@ipfire.org>
6925 Date: Tue, 17 Nov 2020 16:58:55 +0000
6926 Subject: [PATCH 61/70] database: Simplify AS matching code
6927
6928 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6929 ---
6930 src/database.c | 16 ++++++++--------
6931 1 file changed, 8 insertions(+), 8 deletions(-)
6932
6933 diff --git a/src/database.c b/src/database.c
6934 index 1a354f6..be93e00 100644
6935 --- a/src/database.c
6936 +++ b/src/database.c
6937 @@ -1129,12 +1129,6 @@ static int loc_database_enumerator_stack_push_node(
6938 return 0;
6939 }
6940
6941 -static int loc_network_match_asns(struct loc_network* network, struct loc_as_list* asns) {
6942 - uint32_t asn = loc_network_get_asn(network);
6943 -
6944 - return loc_as_list_contains_number(asns, asn);
6945 -}
6946 -
6947 static int loc_database_enumerator_filter_network(
6948 struct loc_database_enumerator* enumerator, struct loc_network* network) {
6949 // Skip if the family does not match
6950 @@ -1152,8 +1146,14 @@ static int loc_database_enumerator_filter_network(
6951 }
6952
6953 // Skip if the ASN does not match
6954 - if (enumerator->asns && !loc_network_match_asns(network, enumerator->asns))
6955 - return 1;
6956 + if (enumerator->asns) {
6957 + if (!loc_as_list_empty(enumerator->asns)) {
6958 + uint32_t asn = loc_network_get_asn(network);
6959 +
6960 + if (!loc_as_list_contains_number(enumerator->asns, asn))
6961 + return 1;
6962 + }
6963 + }
6964
6965 // Skip if flags do not match
6966 if (enumerator->flags &&
6967 --
6968 2.20.1
6969
6970 From d5205091f9cc1ff987e483325d48696459df08d8 Mon Sep 17 00:00:00 2001
6971 From: Michael Tremer <michael.tremer@ipfire.org>
6972 Date: Tue, 17 Nov 2020 17:50:17 +0000
6973 Subject: [PATCH 62/70] countries: Make list grow dynamically
6974
6975 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
6976 ---
6977 src/country-list.c | 38 ++++++++++++++++++++++++++------------
6978 1 file changed, 26 insertions(+), 12 deletions(-)
6979
6980 diff --git a/src/country-list.c b/src/country-list.c
6981 index ae0d71a..1ce2d06 100644
6982 --- a/src/country-list.c
6983 +++ b/src/country-list.c
6984 @@ -25,11 +25,27 @@ struct loc_country_list {
6985 struct loc_ctx* ctx;
6986 int refcount;
6987
6988 - struct loc_country* list[1024];
6989 + struct loc_country** elements;
6990 + size_t elements_size;
6991 +
6992 size_t size;
6993 - size_t max_size;
6994 };
6995
6996 +static int loc_country_list_grow(struct loc_country_list* list, size_t size) {
6997 + DEBUG(list->ctx, "Growing country list %p by %zu to %zu\n",
6998 + list, size, list->elements_size + size);
6999 +
7000 + struct loc_country** elements = reallocarray(list->elements,
7001 + list->elements_size + size, sizeof(*list->elements));
7002 + if (!elements)
7003 + return -errno;
7004 +
7005 + list->elements = elements;
7006 + list->elements_size += size;
7007 +
7008 + return 0;
7009 +}
7010 +
7011 LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
7012 struct loc_country_list** list) {
7013 struct loc_country_list* l = calloc(1, sizeof(*l));
7014 @@ -39,9 +55,6 @@ LOC_EXPORT int loc_country_list_new(struct loc_ctx* ctx,
7015 l->ctx = loc_ref(ctx);
7016 l->refcount = 1;
7017
7018 - // Do not allow this list to grow larger than this
7019 - l->max_size = 1024;
7020 -
7021 DEBUG(l->ctx, "Country list allocated at %p\n", l);
7022 *list = l;
7023
7024 @@ -84,7 +97,7 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
7025
7026 LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
7027 for (unsigned int i = 0; i < list->size; i++)
7028 - loc_country_unref(list->list[i]);
7029 + loc_country_unref(list->elements[i]);
7030 }
7031
7032 LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
7033 @@ -92,7 +105,7 @@ LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* lis
7034 if (index >= list->size)
7035 return NULL;
7036
7037 - return loc_country_ref(list->list[index]);
7038 + return loc_country_ref(list->elements[index]);
7039 }
7040
7041 LOC_EXPORT int loc_country_list_append(
7042 @@ -101,14 +114,15 @@ LOC_EXPORT int loc_country_list_append(
7043 return 0;
7044
7045 // Check if we have space left
7046 - if (list->size == list->max_size) {
7047 - ERROR(list->ctx, "%p: Could not append country to the list. List full\n", list);
7048 - return -ENOMEM;
7049 + if (list->size >= list->elements_size) {
7050 + int r = loc_country_list_grow(list, 64);
7051 + if (r)
7052 + return r;
7053 }
7054
7055 DEBUG(list->ctx, "%p: Appending country %p to list\n", list, country);
7056
7057 - list->list[list->size++] = loc_country_ref(country);
7058 + list->elements[list->size++] = loc_country_ref(country);
7059
7060 return 0;
7061 }
7062 @@ -116,7 +130,7 @@ LOC_EXPORT int loc_country_list_append(
7063 LOC_EXPORT int loc_country_list_contains(
7064 struct loc_country_list* list, struct loc_country* country) {
7065 for (unsigned int i = 0; i < list->size; i++) {
7066 - if (loc_country_cmp(country, list->list[i]) == 0)
7067 + if (loc_country_cmp(country, list->elements[i]) == 0)
7068 return 1;
7069 }
7070
7071 --
7072 2.20.1
7073
7074 From 3b44e4211371d2103f89ba8f056b15edb7778fac Mon Sep 17 00:00:00 2001
7075 From: Michael Tremer <michael.tremer@ipfire.org>
7076 Date: Tue, 17 Nov 2020 17:55:51 +0000
7077 Subject: [PATCH 63/70] networks: Make list grow dynamically
7078
7079 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7080 ---
7081 src/network-list.c | 60 ++++++++++++++++++++++++++++------------------
7082 1 file changed, 37 insertions(+), 23 deletions(-)
7083
7084 diff --git a/src/network-list.c b/src/network-list.c
7085 index 1f6e80e..4912c02 100644
7086 --- a/src/network-list.c
7087 +++ b/src/network-list.c
7088 @@ -25,11 +25,27 @@ struct loc_network_list {
7089 struct loc_ctx* ctx;
7090 int refcount;
7091
7092 - struct loc_network* list[1024];
7093 + struct loc_network** elements;
7094 + size_t elements_size;
7095 +
7096 size_t size;
7097 - size_t max_size;
7098 };
7099
7100 +static int loc_network_list_grow(struct loc_network_list* list, size_t size) {
7101 + DEBUG(list->ctx, "Growing network list %p by %zu to %zu\n",
7102 + list, size, list->elements_size + size);
7103 +
7104 + struct loc_network** elements = reallocarray(list->elements,
7105 + list->elements_size + size, sizeof(*list->elements));
7106 + if (!elements)
7107 + return -errno;
7108 +
7109 + list->elements = elements;
7110 + list->elements_size += size;
7111 +
7112 + return 0;
7113 +}
7114 +
7115 LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
7116 struct loc_network_list** list) {
7117 struct loc_network_list* l = calloc(1, sizeof(*l));
7118 @@ -39,9 +55,6 @@ LOC_EXPORT int loc_network_list_new(struct loc_ctx* ctx,
7119 l->ctx = loc_ref(ctx);
7120 l->refcount = 1;
7121
7122 - // Do not allow this list to grow larger than this
7123 - l->max_size = 1024;
7124 -
7125 DEBUG(l->ctx, "Network list allocated at %p\n", l);
7126 *list = l;
7127 return 0;
7128 @@ -57,7 +70,7 @@ static void loc_network_list_free(struct loc_network_list* list) {
7129 DEBUG(list->ctx, "Releasing network list at %p\n", list);
7130
7131 for (unsigned int i = 0; i < list->size; i++)
7132 - loc_network_unref(list->list[i]);
7133 + loc_network_unref(list->elements[i]);
7134
7135 loc_unref(list->ctx);
7136 free(list);
7137 @@ -84,7 +97,7 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
7138
7139 LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
7140 for (unsigned int i = 0; i < list->size; i++)
7141 - loc_network_unref(list->list[i]);
7142 + loc_network_unref(list->elements[i]);
7143
7144 list->size = 0;
7145 }
7146 @@ -94,7 +107,7 @@ LOC_EXPORT void loc_network_list_dump(struct loc_network_list* list) {
7147 char* s;
7148
7149 for (unsigned int i = 0; i < list->size; i++) {
7150 - network = list->list[i];
7151 + network = list->elements[i];
7152
7153 s = loc_network_str(network);
7154
7155 @@ -108,7 +121,7 @@ LOC_EXPORT struct loc_network* loc_network_list_get(struct loc_network_list* lis
7156 if (index >= list->size)
7157 return NULL;
7158
7159 - return loc_network_ref(list->list[index]);
7160 + return loc_network_ref(list->elements[index]);
7161 }
7162
7163 LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_network* network) {
7164 @@ -117,14 +130,15 @@ LOC_EXPORT int loc_network_list_push(struct loc_network_list* list, struct loc_n
7165 return 0;
7166
7167 // Check if we have space left
7168 - if (list->size == list->max_size) {
7169 - ERROR(list->ctx, "%p: Could not push network onto the stack: Stack full\n", list);
7170 - return -ENOMEM;
7171 + if (list->size >= list->elements_size) {
7172 + int r = loc_network_list_grow(list, 64);
7173 + if (r)
7174 + return r;
7175 }
7176
7177 DEBUG(list->ctx, "%p: Pushing network %p onto stack\n", list, network);
7178
7179 - list->list[list->size++] = loc_network_ref(network);
7180 + list->elements[list->size++] = loc_network_ref(network);
7181
7182 return 0;
7183 }
7184 @@ -136,7 +150,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop(struct loc_network_list* lis
7185 return NULL;
7186 }
7187
7188 - struct loc_network* network = list->list[--list->size];
7189 + struct loc_network* network = list->elements[--list->size];
7190
7191 DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
7192
7193 @@ -150,11 +164,11 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
7194 return NULL;
7195 }
7196
7197 - struct loc_network* network = list->list[0];
7198 + struct loc_network* network = list->elements[0];
7199
7200 // Move all elements to the top of the stack
7201 for (unsigned int i = 0; i < --list->size; i++) {
7202 - list->list[i] = list->list[i+1];
7203 + list->elements[i] = list->elements[i+1];
7204 }
7205
7206 DEBUG(list->ctx, "%p: Popping network %p from stack\n", list, network);
7207 @@ -164,7 +178,7 @@ LOC_EXPORT struct loc_network* loc_network_list_pop_first(struct loc_network_lis
7208
7209 LOC_EXPORT int loc_network_list_contains(struct loc_network_list* list, struct loc_network* network) {
7210 for (unsigned int i = 0; i < list->size; i++) {
7211 - if (loc_network_eq(list->list[i], network))
7212 + if (loc_network_eq(list->elements[i], network))
7213 return 1;
7214 }
7215
7216 @@ -176,11 +190,11 @@ static void loc_network_list_swap(struct loc_network_list* list, unsigned int i1
7217 if (i1 >= list->size || i2 >= list->size)
7218 return;
7219
7220 - struct loc_network* network1 = list->list[i1];
7221 - struct loc_network* network2 = list->list[i2];
7222 + struct loc_network* network1 = list->elements[i1];
7223 + struct loc_network* network2 = list->elements[i2];
7224
7225 - list->list[i1] = network2;
7226 - list->list[i2] = network1;
7227 + list->elements[i1] = network2;
7228 + list->elements[i2] = network1;
7229 }
7230
7231 LOC_EXPORT void loc_network_list_reverse(struct loc_network_list* list) {
7232 @@ -200,7 +214,7 @@ LOC_EXPORT void loc_network_list_sort(struct loc_network_list* list) {
7233 swapped = 0;
7234
7235 for (unsigned int i = 1; i < n; i++) {
7236 - if (loc_network_gt(list->list[i-1], list->list[i]) > 0) {
7237 + if (loc_network_gt(list->elements[i-1], list->elements[i]) > 0) {
7238 loc_network_list_swap(list, i-1, i);
7239 swapped = 1;
7240 }
7241 @@ -215,7 +229,7 @@ LOC_EXPORT int loc_network_list_merge(
7242 int r;
7243
7244 for (unsigned int i = 0; i < other->size; i++) {
7245 - r = loc_network_list_push(self, other->list[i]);
7246 + r = loc_network_list_push(self, other->elements[i]);
7247 if (r)
7248 return r;
7249 }
7250 --
7251 2.20.1
7252
7253 From 1a415f8c555f4fe9a68eb2a897c4a1fc0d33db25 Mon Sep 17 00:00:00 2001
7254 From: Michael Tremer <michael.tremer@ipfire.org>
7255 Date: Tue, 17 Nov 2020 17:57:55 +0000
7256 Subject: [PATCH 64/70] as: Make lists grow dynamically
7257
7258 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7259 ---
7260 src/as-list.c | 38 ++++++++++++++++++++++++++------------
7261 1 file changed, 26 insertions(+), 12 deletions(-)
7262
7263 diff --git a/src/as-list.c b/src/as-list.c
7264 index 7c69eb0..17de23e 100644
7265 --- a/src/as-list.c
7266 +++ b/src/as-list.c
7267 @@ -25,11 +25,27 @@ struct loc_as_list {
7268 struct loc_ctx* ctx;
7269 int refcount;
7270
7271 - struct loc_as* list[1024];
7272 + struct loc_as** elements;
7273 + size_t elements_size;
7274 +
7275 size_t size;
7276 - size_t max_size;
7277 };
7278
7279 +static int loc_as_list_grow(struct loc_as_list* list, size_t size) {
7280 + DEBUG(list->ctx, "Growing AS list %p by %zu to %zu\n",
7281 + list, size, list->elements_size + size);
7282 +
7283 + struct loc_as** elements = reallocarray(list->elements,
7284 + list->elements_size + size, sizeof(*list->elements));
7285 + if (!elements)
7286 + return -errno;
7287 +
7288 + list->elements = elements;
7289 + list->elements_size += size;
7290 +
7291 + return 0;
7292 +}
7293 +
7294 LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
7295 struct loc_as_list** list) {
7296 struct loc_as_list* l = calloc(1, sizeof(*l));
7297 @@ -39,9 +55,6 @@ LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx,
7298 l->ctx = loc_ref(ctx);
7299 l->refcount = 1;
7300
7301 - // Do not allow this list to grow larger than this
7302 - l->max_size = 1024;
7303 -
7304 DEBUG(l->ctx, "AS list allocated at %p\n", l);
7305 *list = l;
7306
7307 @@ -84,7 +97,7 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
7308
7309 LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
7310 for (unsigned int i = 0; i < list->size; i++)
7311 - loc_as_unref(list->list[i]);
7312 + loc_as_unref(list->elements[i]);
7313 }
7314
7315 LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
7316 @@ -92,7 +105,7 @@ LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index
7317 if (index >= list->size)
7318 return NULL;
7319
7320 - return loc_as_ref(list->list[index]);
7321 + return loc_as_ref(list->elements[index]);
7322 }
7323
7324 LOC_EXPORT int loc_as_list_append(
7325 @@ -101,14 +114,15 @@ LOC_EXPORT int loc_as_list_append(
7326 return 0;
7327
7328 // Check if we have space left
7329 - if (list->size == list->max_size) {
7330 - ERROR(list->ctx, "%p: Could not append AS to the list. List full\n", list);
7331 - return -ENOMEM;
7332 + if (list->size >= list->elements_size) {
7333 + int r = loc_as_list_grow(list, 64);
7334 + if (r)
7335 + return r;
7336 }
7337
7338 DEBUG(list->ctx, "%p: Appending AS %p to list\n", list, as);
7339
7340 - list->list[list->size++] = loc_as_ref(as);
7341 + list->elements[list->size++] = loc_as_ref(as);
7342
7343 return 0;
7344 }
7345 @@ -116,7 +130,7 @@ LOC_EXPORT int loc_as_list_append(
7346 LOC_EXPORT int loc_as_list_contains(
7347 struct loc_as_list* list, struct loc_as* as) {
7348 for (unsigned int i = 0; i < list->size; i++) {
7349 - if (loc_as_cmp(as, list->list[i]) == 0)
7350 + if (loc_as_cmp(as, list->elements[i]) == 0)
7351 return 1;
7352 }
7353
7354 --
7355 2.20.1
7356
7357 From e6592434ee7836507c1f436ec3b0db3bc81a81b9 Mon Sep 17 00:00:00 2001
7358 From: Michael Tremer <michael.tremer@ipfire.org>
7359 Date: Tue, 17 Nov 2020 18:13:49 +0000
7360 Subject: [PATCH 65/70] export: Change back to use Network objects
7361
7362 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7363 ---
7364 src/python/export.py | 4 +---
7365 1 file changed, 1 insertion(+), 3 deletions(-)
7366
7367 diff --git a/src/python/export.py b/src/python/export.py
7368 index f675eb3..67e437f 100644
7369 --- a/src/python/export.py
7370 +++ b/src/python/export.py
7371 @@ -144,9 +144,7 @@ class XTGeoIPOutputWriter(OutputWriter):
7372 def _write_network(self, network):
7373 for address in (network.first_address, network.last_address):
7374 # Convert this into a string of bits
7375 - bytes = socket.inet_pton(
7376 - socket.AF_INET6 if network.version == 6 else socket.AF_INET, "%s" % address,
7377 - )
7378 + bytes = socket.inet_pton(network.family, address)
7379
7380 self.f.write(bytes)
7381
7382 --
7383 2.20.1
7384
7385 From 248f5e0419f2349253b8ea96e477c15649fe2173 Mon Sep 17 00:00:00 2001
7386 From: Michael Tremer <michael.tremer@ipfire.org>
7387 Date: Tue, 17 Nov 2020 18:14:15 +0000
7388 Subject: [PATCH 66/70] Actually clear all lists
7389
7390 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7391 ---
7392 src/as-list.c | 8 ++++++++
7393 src/country-list.c | 8 ++++++++
7394 src/network-list.c | 6 ++++++
7395 3 files changed, 22 insertions(+)
7396
7397 diff --git a/src/as-list.c b/src/as-list.c
7398 index 17de23e..76620c7 100644
7399 --- a/src/as-list.c
7400 +++ b/src/as-list.c
7401 @@ -96,8 +96,16 @@ LOC_EXPORT int loc_as_list_empty(struct loc_as_list* list) {
7402 }
7403
7404 LOC_EXPORT void loc_as_list_clear(struct loc_as_list* list) {
7405 + if (!list->elements)
7406 + return;
7407 +
7408 for (unsigned int i = 0; i < list->size; i++)
7409 loc_as_unref(list->elements[i]);
7410 +
7411 + free(list->elements);
7412 + list->elements_size = 0;
7413 +
7414 + list->size = 0;
7415 }
7416
7417 LOC_EXPORT struct loc_as* loc_as_list_get(struct loc_as_list* list, size_t index) {
7418 diff --git a/src/country-list.c b/src/country-list.c
7419 index 1ce2d06..1c49c47 100644
7420 --- a/src/country-list.c
7421 +++ b/src/country-list.c
7422 @@ -96,8 +96,16 @@ LOC_EXPORT int loc_country_list_empty(struct loc_country_list* list) {
7423 }
7424
7425 LOC_EXPORT void loc_country_list_clear(struct loc_country_list* list) {
7426 + if (!list->elements)
7427 + return;
7428 +
7429 for (unsigned int i = 0; i < list->size; i++)
7430 loc_country_unref(list->elements[i]);
7431 +
7432 + free(list->elements);
7433 + list->elements_size = 0;
7434 +
7435 + list->size = 0;
7436 }
7437
7438 LOC_EXPORT struct loc_country* loc_country_list_get(struct loc_country_list* list, size_t index) {
7439 diff --git a/src/network-list.c b/src/network-list.c
7440 index 4912c02..9cb4547 100644
7441 --- a/src/network-list.c
7442 +++ b/src/network-list.c
7443 @@ -96,9 +96,15 @@ LOC_EXPORT int loc_network_list_empty(struct loc_network_list* list) {
7444 }
7445
7446 LOC_EXPORT void loc_network_list_clear(struct loc_network_list* list) {
7447 + if (!list->elements)
7448 + return;
7449 +
7450 for (unsigned int i = 0; i < list->size; i++)
7451 loc_network_unref(list->elements[i]);
7452
7453 + free(list->elements);
7454 + list->elements_size = 0;
7455 +
7456 list->size = 0;
7457 }
7458
7459 --
7460 2.20.1
7461
7462 From c98ebf8aae2aa141193db52cd9429b1ded5b09c4 Mon Sep 17 00:00:00 2001
7463 From: Michael Tremer <michael.tremer@ipfire.org>
7464 Date: Tue, 17 Nov 2020 18:34:51 +0000
7465 Subject: [PATCH 67/70] database: Do not clean up python list
7466
7467 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7468 ---
7469 src/python/database.c | 9 +++++++--
7470 1 file changed, 7 insertions(+), 2 deletions(-)
7471
7472 diff --git a/src/python/database.c b/src/python/database.c
7473 index 38a804c..ed22275 100644
7474 --- a/src/python/database.c
7475 +++ b/src/python/database.c
7476 @@ -325,9 +325,14 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
7477 loc_country_unref(country);
7478 }
7479
7480 - loc_database_enumerator_set_countries(enumerator, countries);
7481 + r = loc_database_enumerator_set_countries(enumerator, countries);
7482 + if (r) {
7483 + PyErr_SetFromErrno(PyExc_SystemError);
7484 +
7485 + loc_as_list_unref(countries);
7486 + return NULL;
7487 + }
7488
7489 - Py_DECREF(country_codes);
7490 loc_country_list_unref(countries);
7491 }
7492
7493 --
7494 2.20.1
7495
7496 From 5470d06cb59027f4e04b6d576763dbf7f1093fde Mon Sep 17 00:00:00 2001
7497 From: Michael Tremer <michael.tremer@ipfire.org>
7498 Date: Tue, 17 Nov 2020 19:01:04 +0000
7499 Subject: [PATCH 68/70] database: Free filter lists in enumerator
7500
7501 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7502 ---
7503 src/database.c | 6 ++++++
7504 1 file changed, 6 insertions(+)
7505
7506 diff --git a/src/database.c b/src/database.c
7507 index be93e00..ca35fe1 100644
7508 --- a/src/database.c
7509 +++ b/src/database.c
7510 @@ -950,6 +950,12 @@ static void loc_database_enumerator_free(struct loc_database_enumerator* enumera
7511 if (enumerator->string)
7512 free(enumerator->string);
7513
7514 + if (enumerator->countries)
7515 + loc_country_list_unref(enumerator->countries);
7516 +
7517 + if (enumerator->asns)
7518 + loc_as_list_unref(enumerator->asns);
7519 +
7520 // Free network search
7521 free(enumerator->networks_visited);
7522
7523 --
7524 2.20.1
7525
7526 From e0e96878d3df51c4a265d51d088005dedf9335e3 Mon Sep 17 00:00:00 2001
7527 From: Michael Tremer <michael.tremer@ipfire.org>
7528 Date: Wed, 18 Nov 2020 13:18:52 +0000
7529 Subject: [PATCH 69/70] database: Add debug output to filtering
7530
7531 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7532 ---
7533 src/database.c | 29 ++++++++++++++++-------------
7534 1 file changed, 16 insertions(+), 13 deletions(-)
7535
7536 diff --git a/src/database.c b/src/database.c
7537 index ca35fe1..83dd752 100644
7538 --- a/src/database.c
7539 +++ b/src/database.c
7540 @@ -1138,33 +1138,36 @@ static int loc_database_enumerator_stack_push_node(
7541 static int loc_database_enumerator_filter_network(
7542 struct loc_database_enumerator* enumerator, struct loc_network* network) {
7543 // Skip if the family does not match
7544 - if (enumerator->family && loc_network_address_family(network) != enumerator->family)
7545 + if (enumerator->family && loc_network_address_family(network) != enumerator->family) {
7546 + DEBUG(enumerator->ctx, "Filtered network %p because of family not matching\n", network);
7547 return 1;
7548 + }
7549
7550 // Skip if the country code does not match
7551 - if (enumerator->countries) {
7552 - if (!loc_country_list_empty(enumerator->countries)) {
7553 - const char* country_code = loc_network_get_country_code(network);
7554 + if (enumerator->countries && !loc_country_list_empty(enumerator->countries)) {
7555 + const char* country_code = loc_network_get_country_code(network);
7556
7557 - if (!loc_country_list_contains_code(enumerator->countries, country_code))
7558 - return 1;
7559 + if (!loc_country_list_contains_code(enumerator->countries, country_code)) {
7560 + DEBUG(enumerator->ctx, "Filtered network %p because of country code not matching\n", network);
7561 + return 1;
7562 }
7563 }
7564
7565 // Skip if the ASN does not match
7566 - if (enumerator->asns) {
7567 - if (!loc_as_list_empty(enumerator->asns)) {
7568 - uint32_t asn = loc_network_get_asn(network);
7569 + if (enumerator->asns && !loc_as_list_empty(enumerator->asns)) {
7570 + uint32_t asn = loc_network_get_asn(network);
7571
7572 - if (!loc_as_list_contains_number(enumerator->asns, asn))
7573 - return 1;
7574 + if (!loc_as_list_contains_number(enumerator->asns, asn)) {
7575 + DEBUG(enumerator->ctx, "Filtered network %p because of ASN not matching\n", network);
7576 + return 1;
7577 }
7578 }
7579
7580 // Skip if flags do not match
7581 - if (enumerator->flags &&
7582 - !loc_network_match_flag(network, enumerator->flags))
7583 + if (enumerator->flags && !loc_network_match_flag(network, enumerator->flags)) {
7584 + DEBUG(enumerator->ctx, "Filtered network %p because of flags not matching\n", network);
7585 return 1;
7586 + }
7587
7588 // Do not filter
7589 return 0;
7590 --
7591 2.20.1
7592
7593 From bce0c9295ff8ff9488f24babe01ce851228d0b1e Mon Sep 17 00:00:00 2001
7594 From: Michael Tremer <michael.tremer@ipfire.org>
7595 Date: Wed, 18 Nov 2020 13:19:04 +0000
7596 Subject: [PATCH 70/70] export: Remove filtering for flags
7597
7598 The filter is an AND filter and if we set the flags from
7599 the special country codes, we won't get back much.
7600
7601 Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
7602 ---
7603 src/python/export.py | 8 +-------
7604 1 file changed, 1 insertion(+), 7 deletions(-)
7605
7606 diff --git a/src/python/export.py b/src/python/export.py
7607 index 67e437f..4219957 100644
7608 --- a/src/python/export.py
7609 +++ b/src/python/export.py
7610 @@ -187,15 +187,9 @@ class Exporter(object):
7611 country_code for country_code in countries if not country_code in FLAGS.values()
7612 ]
7613
7614 - # Collect flags
7615 - flags = 0
7616 - for flag in FLAGS:
7617 - if FLAGS[flag] in countries:
7618 - flags |= flag
7619 -
7620 # Get all networks that match the family
7621 networks = self.db.search_networks(family=family,
7622 - country_codes=country_codes, asns=asns, flags=flags, flatten=True)
7623 + country_codes=country_codes, asns=asns, flatten=True)
7624
7625 # Walk through all networks
7626 for network in networks:
7627 --
7628 2.20.1
7629
Peter's tree of IPFire 2.x