]>
Commit | Line | Data |
---|---|---|
88ef7e9c MT |
1 | #!/usr/bin/python3 |
2 | ############################################################################### | |
3 | # # | |
4 | # libloc - A library to determine the location of someone on the Internet # | |
5 | # # | |
e17e804e | 6 | # Copyright (C) 2020-2021 IPFire Development Team <info@ipfire.org> # |
88ef7e9c MT |
7 | # # |
8 | # This library is free software; you can redistribute it and/or # | |
9 | # modify it under the terms of the GNU Lesser General Public # | |
10 | # License as published by the Free Software Foundation; either # | |
11 | # version 2.1 of the License, or (at your option) any later version. # | |
12 | # # | |
13 | # This library is distributed in the hope that it will be useful, # | |
14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of # | |
15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # | |
16 | # Lesser General Public License for more details. # | |
17 | # # | |
18 | ############################################################################### | |
19 | ||
20 | import io | |
21 | import ipaddress | |
22 | import logging | |
47de14b0 | 23 | import math |
88ef7e9c MT |
24 | import os |
25 | import socket | |
26 | ||
fae36e81 MT |
27 | import _location |
28 | ||
88ef7e9c MT |
29 | # Initialise logging |
30 | log = logging.getLogger("location.export") | |
31 | log.propagate = 1 | |
32 | ||
bd1dc6bf | 33 | FLAGS = { |
fae36e81 MT |
34 | _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1", |
35 | _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2", | |
36 | _location.NETWORK_FLAG_ANYCAST : "A3", | |
e17e804e | 37 | _location.NETWORK_FLAG_DROP : "XD", |
fae36e81 MT |
38 | } |
39 | ||
88ef7e9c MT |
40 | class OutputWriter(object): |
41 | suffix = "networks" | |
42 | mode = "w" | |
43 | ||
27dc4fa5 MT |
44 | def __init__(self, f, family=None, prefix=None): |
45 | self.f = f | |
46 | self.prefix = prefix | |
47 | self.family = family | |
88ef7e9c | 48 | |
47de14b0 MT |
49 | # Call any custom initialization |
50 | self.init() | |
51 | ||
88ef7e9c MT |
52 | # Immediately write the header |
53 | self._write_header() | |
54 | ||
47de14b0 MT |
55 | def init(self): |
56 | """ | |
57 | To be overwritten by anything that inherits from this | |
58 | """ | |
59 | pass | |
60 | ||
88ef7e9c | 61 | @classmethod |
27dc4fa5 | 62 | def open(cls, filename, *args, **kwargs): |
88ef7e9c MT |
63 | """ |
64 | Convenience function to open a file | |
65 | """ | |
66 | f = open(filename, cls.mode) | |
67 | ||
27dc4fa5 | 68 | return cls(f, *args, **kwargs) |
88ef7e9c MT |
69 | |
70 | def __repr__(self): | |
71 | return "<%s f=%s>" % (self.__class__.__name__, self.f) | |
72 | ||
73 | def _write_header(self): | |
74 | """ | |
75 | The header of the file | |
76 | """ | |
77 | pass | |
78 | ||
79 | def _write_footer(self): | |
80 | """ | |
81 | The footer of the file | |
82 | """ | |
83 | pass | |
84 | ||
c242f732 | 85 | def write(self, network): |
90d2194a | 86 | self.f.write("%s\n" % network) |
43554dc4 | 87 | |
88ef7e9c MT |
88 | def finish(self): |
89 | """ | |
90 | Called when all data has been written | |
91 | """ | |
92 | self._write_footer() | |
93 | ||
94 | # Close the file | |
95 | self.f.close() | |
96 | ||
97 | ||
98 | class IpsetOutputWriter(OutputWriter): | |
99 | """ | |
100 | For ipset | |
101 | """ | |
102 | suffix = "ipset" | |
103 | ||
47de14b0 MT |
104 | # The value is being used if we don't know any better |
105 | DEFAULT_HASHSIZE = 64 | |
106 | ||
107 | # We aim for this many networks in a bucket on average. This allows us to choose | |
108 | # how much memory we want to sacrifice to gain better performance. The lower the | |
109 | # factor, the faster a lookup will be, but it will use more memory. | |
110 | # We will aim for only using three quarters of all buckets to avoid any searches | |
111 | # through the linked lists. | |
112 | HASHSIZE_FACTOR = 0.75 | |
113 | ||
114 | def init(self): | |
115 | # Count all networks | |
116 | self.networks = 0 | |
117 | ||
118 | @property | |
119 | def hashsize(self): | |
120 | """ | |
121 | Calculates an optimized hashsize | |
122 | """ | |
123 | # Return the default value if we don't know the size of the set | |
124 | if not self.networks: | |
125 | return self.DEFAULT_HASHSIZE | |
126 | ||
127 | # Find the nearest power of two that is larger than the number of networks | |
128 | # divided by the hashsize factor. | |
129 | exponent = math.log(self.networks / self.HASHSIZE_FACTOR, 2) | |
130 | ||
131 | # Return the size of the hash | |
132 | return 2 ** math.ceil(exponent) | |
133 | ||
88ef7e9c | 134 | def _write_header(self): |
47de14b0 | 135 | # This must have a fixed size, because we will write the header again in the end |
27dc4fa5 MT |
136 | self.f.write("create %s hash:net family inet%s" % ( |
137 | self.prefix, | |
138 | "6" if self.family == socket.AF_INET6 else "" | |
139 | )) | |
52176cc7 | 140 | self.f.write(" hashsize %8d maxelem 1048576 -exist\n" % self.hashsize) |
1b759b42 | 141 | self.f.write("flush %s\n" % self.prefix) |
88ef7e9c | 142 | |
90d2194a | 143 | def write(self, network): |
88ef7e9c MT |
144 | self.f.write("add %s %s\n" % (self.prefix, network)) |
145 | ||
47de14b0 MT |
146 | # Increment network counter |
147 | self.networks += 1 | |
148 | ||
149 | def _write_footer(self): | |
150 | # Jump back to the beginning of the file | |
151 | self.f.seek(0) | |
152 | ||
153 | # Rewrite the header with better configuration | |
154 | self._write_header() | |
155 | ||
88ef7e9c MT |
156 | |
157 | class NftablesOutputWriter(OutputWriter): | |
158 | """ | |
159 | For nftables | |
160 | """ | |
161 | suffix = "set" | |
162 | ||
163 | def _write_header(self): | |
164 | self.f.write("define %s = {\n" % self.prefix) | |
165 | ||
166 | def _write_footer(self): | |
167 | self.f.write("}\n") | |
168 | ||
90d2194a | 169 | def write(self, network): |
88ef7e9c MT |
170 | self.f.write(" %s,\n" % network) |
171 | ||
172 | ||
173 | class XTGeoIPOutputWriter(OutputWriter): | |
174 | """ | |
175 | Formats the output in that way, that it can be loaded by | |
176 | the xt_geoip kernel module from xtables-addons. | |
177 | """ | |
178 | suffix = "iv" | |
179 | mode = "wb" | |
180 | ||
90d2194a | 181 | def write(self, network): |
90188dad MT |
182 | self.f.write(network._first_address) |
183 | self.f.write(network._last_address) | |
88ef7e9c MT |
184 | |
185 | ||
186 | formats = { | |
187 | "ipset" : IpsetOutputWriter, | |
188 | "list" : OutputWriter, | |
189 | "nftables" : NftablesOutputWriter, | |
190 | "xt_geoip" : XTGeoIPOutputWriter, | |
191 | } | |
192 | ||
193 | class Exporter(object): | |
194 | def __init__(self, db, writer): | |
195 | self.db, self.writer = db, writer | |
196 | ||
197 | def export(self, directory, families, countries, asns): | |
198 | for family in families: | |
199 | log.debug("Exporting family %s" % family) | |
200 | ||
201 | writers = {} | |
202 | ||
203 | # Create writers for countries | |
204 | for country_code in countries: | |
205 | filename = self._make_filename( | |
206 | directory, prefix=country_code, suffix=self.writer.suffix, family=family, | |
207 | ) | |
208 | ||
27dc4fa5 | 209 | writers[country_code] = self.writer.open(filename, family, prefix="%s" % country_code) |
88ef7e9c MT |
210 | |
211 | # Create writers for ASNs | |
212 | for asn in asns: | |
213 | filename = self._make_filename( | |
214 | directory, "AS%s" % asn, suffix=self.writer.suffix, family=family, | |
215 | ) | |
216 | ||
27dc4fa5 | 217 | writers[asn] = self.writer.open(filename, family, prefix="AS%s" % asn) |
88ef7e9c | 218 | |
7af51f8a MT |
219 | # Filter countries from special country codes |
220 | country_codes = [ | |
bd1dc6bf | 221 | country_code for country_code in countries if not country_code in FLAGS.values() |
7af51f8a MT |
222 | ] |
223 | ||
88ef7e9c | 224 | # Get all networks that match the family |
7af51f8a | 225 | networks = self.db.search_networks(family=family, |
bce0c929 | 226 | country_codes=country_codes, asns=asns, flatten=True) |
28c73fa3 | 227 | |
88ef7e9c | 228 | # Walk through all networks |
bbed1fd2 | 229 | for network in networks: |
88ef7e9c | 230 | # Write matching countries |
c242f732 MT |
231 | try: |
232 | writers[network.country_code].write(network) | |
233 | except KeyError: | |
234 | pass | |
88ef7e9c MT |
235 | |
236 | # Write matching ASNs | |
c242f732 MT |
237 | try: |
238 | writers[network.asn].write(network) | |
239 | except KeyError: | |
240 | pass | |
88ef7e9c | 241 | |
fae36e81 | 242 | # Handle flags |
bd1dc6bf | 243 | for flag in FLAGS: |
fae36e81 MT |
244 | if network.has_flag(flag): |
245 | # Fetch the "fake" country code | |
bd1dc6bf | 246 | country = FLAGS[flag] |
fae36e81 | 247 | |
c242f732 MT |
248 | try: |
249 | writers[country].write(network) | |
250 | except KeyError: | |
251 | pass | |
fae36e81 | 252 | |
88ef7e9c MT |
253 | # Write everything to the filesystem |
254 | for writer in writers.values(): | |
255 | writer.finish() | |
256 | ||
257 | def _make_filename(self, directory, prefix, suffix, family): | |
258 | filename = "%s.%s%s" % ( | |
259 | prefix, suffix, "6" if family == socket.AF_INET6 else "4" | |
260 | ) | |
261 | ||
262 | return os.path.join(directory, filename) |