]>
Commit | Line | Data |
---|---|---|
88ef7e9c MT |
1 | ############################################################################### |
2 | # # | |
3 | # libloc - A library to determine the location of someone on the Internet # | |
4 | # # | |
e17e804e | 5 | # Copyright (C) 2020-2021 IPFire Development Team <info@ipfire.org> # |
88ef7e9c MT |
6 | # # |
7 | # This library is free software; you can redistribute it and/or # | |
8 | # modify it under the terms of the GNU Lesser General Public # | |
9 | # License as published by the Free Software Foundation; either # | |
10 | # version 2.1 of the License, or (at your option) any later version. # | |
11 | # # | |
12 | # This library is distributed in the hope that it will be useful, # | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of # | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # | |
15 | # Lesser General Public License for more details. # | |
16 | # # | |
17 | ############################################################################### | |
18 | ||
19 | import io | |
20 | import ipaddress | |
21 | import logging | |
47de14b0 | 22 | import math |
88ef7e9c MT |
23 | import os |
24 | import socket | |
58f0922d | 25 | import sys |
88ef7e9c | 26 | |
58f0922d | 27 | from .i18n import _ |
fae36e81 MT |
28 | import _location |
29 | ||
88ef7e9c MT |
30 | # Initialise logging |
31 | log = logging.getLogger("location.export") | |
32 | log.propagate = 1 | |
33 | ||
bd1dc6bf | 34 | FLAGS = { |
fae36e81 MT |
35 | _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", |
36 | _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", | |
37 | _location.NETWORK_FLAG_ANYCAST : "A3", | |
e17e804e | 38 | _location.NETWORK_FLAG_DROP : "XD", |
fae36e81 MT |
39 | } |
40 | ||
88ef7e9c MT |
41 | class OutputWriter(object): |
42 | suffix = "networks" | |
43 | mode = "w" | |
44 | ||
f1fb21bd | 45 | def __init__(self, name, family=None, directory=None, f=None): |
ce1e53c7 | 46 | self.name = name |
27dc4fa5 | 47 | self.family = family |
ce1e53c7 MT |
48 | self.directory = directory |
49 | ||
cd214f29 MT |
50 | # Tag |
51 | self.tag = self._make_tag() | |
52 | ||
ce1e53c7 | 53 | # Open output file |
f1fb21bd MT |
54 | if f: |
55 | self.f = f | |
56 | elif self.directory: | |
ce1e53c7 MT |
57 | self.f = open(self.filename, self.mode) |
58 | elif "b" in self.mode: | |
59 | self.f = io.BytesIO() | |
60 | else: | |
61 | self.f = io.StringIO() | |
88ef7e9c | 62 | |
47de14b0 MT |
63 | # Call any custom initialization |
64 | self.init() | |
65 | ||
88ef7e9c MT |
66 | # Immediately write the header |
67 | self._write_header() | |
68 | ||
47de14b0 MT |
69 | def init(self): |
70 | """ | |
71 | To be overwritten by anything that inherits from this | |
72 | """ | |
73 | pass | |
74 | ||
ce1e53c7 MT |
75 | def __repr__(self): |
76 | return "<%s %s f=%s>" % (self.__class__.__name__, self, self.f) | |
88ef7e9c | 77 | |
1cd4ddbc | 78 | def _make_tag(self): |
ce1e53c7 MT |
79 | families = { |
80 | socket.AF_INET6 : "6", | |
81 | socket.AF_INET : "4", | |
82 | } | |
88ef7e9c | 83 | |
ce1e53c7 MT |
84 | return "%sv%s" % (self.name, families.get(self.family, "?")) |
85 | ||
fbe0f743 | 86 | @property |
ce1e53c7 MT |
87 | def filename(self): |
88 | if self.directory: | |
89 | return os.path.join(self.directory, "%s.%s" % (self.tag, self.suffix)) | |
88ef7e9c MT |
90 | |
91 | def _write_header(self): | |
92 | """ | |
93 | The header of the file | |
94 | """ | |
95 | pass | |
96 | ||
97 | def _write_footer(self): | |
98 | """ | |
99 | The footer of the file | |
100 | """ | |
101 | pass | |
102 | ||
c242f732 | 103 | def write(self, network): |
90d2194a | 104 | self.f.write("%s\n" % network) |
43554dc4 | 105 | |
88ef7e9c MT |
106 | def finish(self): |
107 | """ | |
108 | Called when all data has been written | |
109 | """ | |
110 | self._write_footer() | |
111 | ||
58f0922d MT |
112 | # Flush all output |
113 | self.f.flush() | |
114 | ||
115 | def print(self): | |
116 | """ | |
117 | Prints the entire output line by line | |
118 | """ | |
119 | if isinstance(self.f, io.BytesIO): | |
120 | raise TypeError(_("Won't write binary output to stdout")) | |
121 | ||
122 | # Go back to the beginning | |
123 | self.f.seek(0) | |
124 | ||
125 | # Iterate over everything line by line | |
126 | for line in self.f: | |
127 | sys.stdout.write(line) | |
88ef7e9c MT |
128 | |
129 | ||
130 | class IpsetOutputWriter(OutputWriter): | |
131 | """ | |
132 | For ipset | |
133 | """ | |
134 | suffix = "ipset" | |
135 | ||
47de14b0 MT |
136 | # The value is being used if we don't know any better |
137 | DEFAULT_HASHSIZE = 64 | |
138 | ||
139 | # We aim for this many networks in a bucket on average. This allows us to choose | |
140 | # how much memory we want to sacrifice to gain better performance. The lower the | |
141 | # factor, the faster a lookup will be, but it will use more memory. | |
142 | # We will aim for only using three quarters of all buckets to avoid any searches | |
143 | # through the linked lists. | |
144 | HASHSIZE_FACTOR = 0.75 | |
145 | ||
146 | def init(self): | |
147 | # Count all networks | |
148 | self.networks = 0 | |
149 | ||
1f2ece28 MT |
150 | # Check that family is being set |
151 | if not self.family: | |
152 | raise ValueError("%s requires family being set" % self.__class__.__name__) | |
153 | ||
47de14b0 MT |
154 | @property |
155 | def hashsize(self): | |
156 | """ | |
157 | Calculates an optimized hashsize | |
158 | """ | |
159 | # Return the default value if we don't know the size of the set | |
160 | if not self.networks: | |
161 | return self.DEFAULT_HASHSIZE | |
162 | ||
163 | # Find the nearest power of two that is larger than the number of networks | |
164 | # divided by the hashsize factor. | |
165 | exponent = math.log(self.networks / self.HASHSIZE_FACTOR, 2) | |
166 | ||
f1f8927a MT |
167 | # Return the size of the hash (the minimum is 64) |
168 | return max(2 ** math.ceil(exponent), 64) | |
47de14b0 | 169 | |
88ef7e9c | 170 | def _write_header(self): |
47de14b0 | 171 | # This must have a fixed size, because we will write the header again in the end |
27dc4fa5 | 172 | self.f.write("create %s hash:net family inet%s" % ( |
ce1e53c7 | 173 | self.tag, |
27dc4fa5 MT |
174 | "6" if self.family == socket.AF_INET6 else "" |
175 | )) | |
52176cc7 | 176 | self.f.write(" hashsize %8d maxelem 1048576 -exist\n" % self.hashsize) |
ce1e53c7 | 177 | self.f.write("flush %s\n" % self.tag) |
88ef7e9c | 178 | |
90d2194a | 179 | def write(self, network): |
ce1e53c7 | 180 | self.f.write("add %s %s\n" % (self.tag, network)) |
88ef7e9c | 181 | |
47de14b0 MT |
182 | # Increment network counter |
183 | self.networks += 1 | |
184 | ||
185 | def _write_footer(self): | |
186 | # Jump back to the beginning of the file | |
a4c22636 MT |
187 | try: |
188 | self.f.seek(0) | |
189 | ||
190 | # If the output stream isn't seekable, we won't try writing the header again | |
191 | except io.UnsupportedOperation: | |
192 | return | |
47de14b0 MT |
193 | |
194 | # Rewrite the header with better configuration | |
195 | self._write_header() | |
196 | ||
88ef7e9c MT |
197 | |
198 | class NftablesOutputWriter(OutputWriter): | |
199 | """ | |
200 | For nftables | |
201 | """ | |
202 | suffix = "set" | |
203 | ||
204 | def _write_header(self): | |
ce1e53c7 | 205 | self.f.write("define %s = {\n" % self.tag) |
88ef7e9c MT |
206 | |
207 | def _write_footer(self): | |
208 | self.f.write("}\n") | |
209 | ||
90d2194a | 210 | def write(self, network): |
88ef7e9c MT |
211 | self.f.write(" %s,\n" % network) |
212 | ||
213 | ||
214 | class XTGeoIPOutputWriter(OutputWriter): | |
215 | """ | |
216 | Formats the output in that way, that it can be loaded by | |
217 | the xt_geoip kernel module from xtables-addons. | |
218 | """ | |
88ef7e9c MT |
219 | mode = "wb" |
220 | ||
ce1e53c7 MT |
221 | @property |
222 | def tag(self): | |
223 | return self.name | |
224 | ||
225 | @property | |
226 | def suffix(self): | |
227 | return "iv%s" % ("6" if self.family == socket.AF_INET6 else "4") | |
228 | ||
90d2194a | 229 | def write(self, network): |
90188dad MT |
230 | self.f.write(network._first_address) |
231 | self.f.write(network._last_address) | |
88ef7e9c MT |
232 | |
233 | ||
234 | formats = { | |
235 | "ipset" : IpsetOutputWriter, | |
236 | "list" : OutputWriter, | |
237 | "nftables" : NftablesOutputWriter, | |
238 | "xt_geoip" : XTGeoIPOutputWriter, | |
239 | } | |
240 | ||
241 | class Exporter(object): | |
242 | def __init__(self, db, writer): | |
243 | self.db, self.writer = db, writer | |
244 | ||
245 | def export(self, directory, families, countries, asns): | |
246 | for family in families: | |
247 | log.debug("Exporting family %s" % family) | |
248 | ||
249 | writers = {} | |
250 | ||
251 | # Create writers for countries | |
252 | for country_code in countries: | |
ce1e53c7 | 253 | writers[country_code] = self.writer(country_code, family=family, directory=directory) |
88ef7e9c MT |
254 | |
255 | # Create writers for ASNs | |
256 | for asn in asns: | |
ce1e53c7 | 257 | writers[asn] = self.writer("AS%s" % asn, family=family, directory=directory) |
88ef7e9c | 258 | |
7af51f8a MT |
259 | # Filter countries from special country codes |
260 | country_codes = [ | |
bd1dc6bf | 261 | country_code for country_code in countries if not country_code in FLAGS.values() |
7af51f8a MT |
262 | ] |
263 | ||
88ef7e9c | 264 | # Get all networks that match the family |
7af51f8a | 265 | networks = self.db.search_networks(family=family, |
a3140aa9 | 266 | country_codes=country_codes, asns=asns, flatten=True) |
28c73fa3 | 267 | |
88ef7e9c | 268 | # Walk through all networks |
bbed1fd2 | 269 | for network in networks: |
88ef7e9c | 270 | # Write matching countries |
c242f732 MT |
271 | try: |
272 | writers[network.country_code].write(network) | |
273 | except KeyError: | |
274 | pass | |
88ef7e9c MT |
275 | |
276 | # Write matching ASNs | |
c242f732 MT |
277 | try: |
278 | writers[network.asn].write(network) | |
279 | except KeyError: | |
280 | pass | |
88ef7e9c | 281 | |
fae36e81 | 282 | # Handle flags |
bd1dc6bf | 283 | for flag in FLAGS: |
fae36e81 MT |
284 | if network.has_flag(flag): |
285 | # Fetch the "fake" country code | |
bd1dc6bf | 286 | country = FLAGS[flag] |
fae36e81 | 287 | |
c242f732 MT |
288 | try: |
289 | writers[country].write(network) | |
290 | except KeyError: | |
291 | pass | |
fae36e81 | 292 | |
88ef7e9c MT |
293 | # Write everything to the filesystem |
294 | for writer in writers.values(): | |
295 | writer.finish() | |
296 | ||
58f0922d MT |
297 | # Print to stdout |
298 | if not directory: | |
299 | for writer in writers.values(): | |
300 | writer.print() |