]> git.ipfire.org Git - ipfire.org.git/commitdiff
Use Python's internal asyncio stuff instead of Tornado's
authorMichael Tremer <michael.tremer@ipfire.org>
Tue, 5 Nov 2019 16:35:55 +0000 (16:35 +0000)
committerMichael Tremer <michael.tremer@ipfire.org>
Tue, 5 Nov 2019 16:35:55 +0000 (16:35 +0000)
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
16 files changed:
src/backend/accounts.py
src/backend/base.py
src/backend/blog.py
src/backend/geoip.py
src/backend/messages.py
src/backend/mirrors.py
src/backend/releases.py
src/backend/tweets.py
src/backend/wiki.py
src/backend/zeiterfassung.py
src/scripts/ipfire.org.in
src/web/auth.py
src/web/base.py
src/web/donate.py
src/web/location.py
src/web/newsletter.py

index f79b4fd394dda08bdf3cebb6394829f3e5af1322..18ae2d3194a7e9d1c5248358f0e221d78f6dd6b8 100644 (file)
@@ -180,12 +180,11 @@ class Accounts(Object):
                        "(&(objectClass=inetOrgPerson)(|(sipAuthenticationUser=%s)(telephoneNumber=%s)(homePhone=%s)(mobile=%s)))" \
                        % (number, number, number, number))
 
-       @tornado.gen.coroutine
-       def check_spam(self, uid, email, address):
+       async def check_spam(self, uid, email, address):
                sfs = StopForumSpam(self.backend, uid, email, address)
 
                # Get spam score
-               score = yield sfs.check()
+               score = await sfs.check()
 
                return score >= 50
 
@@ -1051,8 +1050,7 @@ class StopForumSpam(Object):
        def init(self, uid, email, address):
                self.uid, self.email, self.address = uid, email, address
 
-       @tornado.gen.coroutine
-       def send_request(self, **kwargs):
+       async def send_request(self, **kwargs):
                arguments = {
                        "json" : "1",
                }
@@ -1064,14 +1062,13 @@ class StopForumSpam(Object):
                request.body = urllib.parse.urlencode(arguments)
 
                # Send the request
-               response = yield self.backend.http_client.fetch(request)
+               response = await self.backend.http_client.fetch(request)
 
                # Decode the JSON response
                return json.loads(response.body.decode())
 
-       @tornado.gen.coroutine
-       def check_address(self):
-               response = yield self.send_request(ip=self.address)
+       async def check_address(self):
+               response = await self.send_request(ip=self.address)
 
                try:
                        confidence = response["ip"]["confidence"]
@@ -1082,9 +1079,8 @@ class StopForumSpam(Object):
 
                return confidence
 
-       @tornado.gen.coroutine
-       def check_username(self):
-               response = yield self.send_request(username=self.uid)
+       async def check_username(self):
+               response = await self.send_request(username=self.uid)
 
                try:
                        confidence = response["username"]["confidence"]
@@ -1095,9 +1091,8 @@ class StopForumSpam(Object):
 
                return confidence
 
-       @tornado.gen.coroutine
-       def check_email(self):
-               response = yield self.send_request(email=self.email)
+       async def check_email(self):
+               response = await self.send_request(email=self.email)
 
                try:
                        confidence = response["email"]["confidence"]
@@ -1108,8 +1103,7 @@ class StopForumSpam(Object):
 
                return confidence
 
-       @tornado.gen.coroutine
-       def check(self, threshold=95):
+       async def check(self, threshold=95):
                """
                        This function tries to detect if we have a spammer.
 
@@ -1117,10 +1111,10 @@ class StopForumSpam(Object):
                        address and username and if those are on the database, we
                        will send the email address as well.
                """
-               confidences = yield [self.check_address(), self.check_username()]
+               confidences = [await self.check_address(), await self.check_username()]
 
                if any((c < threshold for c in confidences)):
-                       confidences += yield [self.check_email()]
+                       confidences.append(await self.check_email())
 
                # Build a score based on the lowest confidence
                return 100 - min(confidences)
index ebd74a00b70345271e7500cc8077b34e5a920912..f85d2ad891c88c112e57e95235dceabeb2c7cb3b 100644 (file)
@@ -2,7 +2,6 @@
 
 import configparser
 import io
-import tornado.gen
 import tornado.httpclient
 
 from . import accounts
@@ -94,8 +93,7 @@ class Backend(object):
 
                self.db = database.Connection(**credentials)
 
-       @tornado.gen.coroutine
-       def run_task(self, task, *args, **kwargs):
+       async def run_task(self, task, *args, **kwargs):
                tasks = {
                        "check-mirrors"     : self.mirrors.check_all,
                        "check-spam"        : self.accounts.check_spam,
@@ -114,7 +112,7 @@ class Backend(object):
                        raise ValueError("Unknown task: %s" % task)
 
                # Run the task
-               r = yield func(*args, **kwargs)
+               r = await func(*args, **kwargs)
 
                # If any error code has been returned,
                # we will end the program
@@ -137,8 +135,7 @@ class Backend(object):
        def tweets(self):
                return tweets.Tweets(self)
 
-       @tornado.gen.coroutine
-       def cleanup(self):
+       async def cleanup(self):
                # Cleanup message queue
                with self.db.transaction():
                        self.messages.queue.cleanup()
index c66cc37aea9670f3d4331be51c12d87b12e94551..61f4ba3f83827274d82d75d91883c69dd24b3155 100644 (file)
@@ -7,7 +7,6 @@ import markdown.extensions
 import markdown.preprocessors
 import re
 import textile
-import tornado.gen
 import unicodedata
 
 from . import misc
@@ -166,8 +165,7 @@ class Blog(misc.Object):
                for row in res:
                        yield row.year
 
-       @tornado.gen.coroutine
-       def update_feeds(self):
+       async def update_feeds(self):
                """
                        Updates all enabled feeds
                """
index 9ee79cc4e6c2e33877becfcfd9e8f49f25ee5532..dffafa4f70b6e2c5357daf40d137ae197daf5968 100644 (file)
@@ -42,8 +42,7 @@ class Resolver(tornado.platform.caresresolver.CaresResolver):
                # Overwrite Channel
                self.channel = pycares.Channel(sock_state_cb=self._sock_state_cb, **kwargs)
 
-       @tornado.gen.coroutine
-       def query(self, name, type=pycares.QUERY_TYPE_A):
+       async def query(self, name, type=pycares.QUERY_TYPE_A):
                # Create a new Future
                fut = tornado.gen.Future()
 
@@ -51,7 +50,7 @@ class Resolver(tornado.platform.caresresolver.CaresResolver):
                self.channel.query(name, type, lambda result, error: fut.set_result((result, error)))
 
                # Wait for the response
-               result, error = yield fut
+               result, error = await fut
 
                # Handle any errors
                if error:
@@ -125,12 +124,11 @@ class GeoIP(Object):
        def get_country_name(self, code):
                return countries.get_name(code)
 
-       @tornado.gen.coroutine
-       def test_blacklist(self, address):
+       async def test_blacklist(self, address):
                address = self.lookup(address)
 
                # Determne blacklist status
-               status = yield address.is_blacklisted()
+               status = await address.is_blacklisted()
 
                print("Blacklist status for %s: %s" % (address, status))
 
@@ -167,8 +165,7 @@ class Address(Object):
 
                return ".".join(octets)
 
-       @tornado.gen.coroutine
-       def _resolve_blacklist(self, blacklist):
+       async def _resolve_blacklist(self, blacklist):
                return_code = None
 
                # Get resource record name
@@ -184,7 +181,7 @@ class Address(Object):
 
                # Run query
                try:
-                       res = yield self.backend.geoip.resolver.query(rr, type=type)
+                       res = await self.backend.geoip.resolver.query(rr, type=type)
                except IOError as e:
                        logging.warning(e)
 
@@ -201,7 +198,7 @@ class Address(Object):
                        break
 
                # If the IP address is on a blacklist, we will try to fetch the TXT record
-               reason = yield self.backend.geoip.resolver.query(rr, type=pycares.QUERY_TYPE_TXT)
+               reason = await self.backend.geoip.resolver.query(rr, type=pycares.QUERY_TYPE_TXT)
 
                # Log result
                logging.debug("%s is blacklisted on %s: %s" % (self, blacklist, reason or "N/A"))
@@ -214,22 +211,20 @@ class Address(Object):
                # Blocked, but no reason
                return return_code, None
 
-       @tornado.gen.coroutine
-       def get_blacklists(self):
-               blacklists = yield { bl : self._resolve_blacklist(bl) for bl in BLACKLISTS }
+       async def get_blacklists(self):
+               blacklists = { bl : self._resolve_blacklist(bl) for bl in BLACKLISTS }
 
                return blacklists
 
-       @tornado.gen.coroutine
-       def is_blacklisted(self):
+       async def is_blacklisted(self):
                logging.debug("Checking if %s is blacklisted..." % self)
 
                # Perform checks
-               blacklists = yield { bl : self._resolve_blacklist(bl) for bl in BLOCKLISTS }
+               blacklists = { bl : self._resolve_blacklist(bl) for bl in BLOCKLISTS }
 
                # If we are blacklisted on one list, this one is screwed
                for bl in blacklists:
-                       code, message = blacklists[bl]
+                       code, message = await blacklists[bl]
 
                        logging.debug("Response from %s is: %s (%s)" % (bl, code, message))
 
index ed2d896b5e422be1006883976809f074a1f839d9..1b0e85a2bd47ecb6e51b9c1a77e8b1e0a2547620 100644 (file)
@@ -6,7 +6,6 @@ import email.mime.nonmultipart
 import email.utils
 import logging
 import subprocess
-import tornado.gen
 import tornado.template
 
 from . import accounts
@@ -151,8 +150,7 @@ class Queue(misc.Object):
                        WHERE time_sent IS NULL \
                                ORDER BY priority DESC, time_created ASC")
 
-       @tornado.gen.coroutine
-       def send_all(self):
+       async def send_all(self):
                # Sends all messages
                for message in self.messages:
                        self._sendmail(message)
index 647bbf2e95d33205db62129b8cb1043cee2d11d3..50ceb34d585611889c4b0911051c1dc2d0caabec 100644 (file)
@@ -8,7 +8,6 @@ import random
 import socket
 import time
 import ssl
-import tornado.gen
 import tornado.httpclient
 import tornado.iostream
 import tornado.netutil
@@ -37,11 +36,10 @@ class Mirrors(Object):
 
                return iter(mirrors)
 
-       @tornado.gen.coroutine
-       def check_all(self):
+       async def check_all(self):
                for mirror in self:
                        with self.db.transaction():
-                               yield mirror.check()
+                               await mirror.check()
 
        def get_for_download(self, filename, country_code=None):
                # Try to find a good mirror for this country first
@@ -197,17 +195,15 @@ class Mirror(Object):
        def disabled(self):
                return not self.enabled
 
-       @tornado.gen.coroutine
-       def check(self):
+       async def check(self):
                logging.debug("Running check for mirror %s" % self.hostname)
 
                self.db.execute("UPDATE mirrors SET address = %s WHERE id = %s",
                        self.address, self.id)
 
-               success = yield self.check_timestamp()
-
+               success = await self.check_timestamp()
                if success:
-                       yield self.check_filelist()
+                       await self.check_filelist()
 
        def check_state(self, last_update):
                logging.debug("Checking state of mirror %s" % self.id)
@@ -233,12 +229,11 @@ class Mirror(Object):
 
                self.set_state("UP")
 
-       @tornado.gen.coroutine
-       def check_timestamp(self):
+       async def check_timestamp(self):
                http = tornado.httpclient.AsyncHTTPClient()
 
                try:
-                       response = yield http.fetch(self.url + ".timestamp",
+                       response = await http.fetch(self.url + ".timestamp",
                                headers={ "Pragma" : "no-cache" })
                except tornado.httpclient.HTTPError as e:
                        logging.error("Error getting timestamp from %s: %s" % (self.hostname, e))
@@ -282,8 +277,7 @@ class Mirror(Object):
 
                return True
 
-       @tornado.gen.coroutine
-       def check_filelist(self):
+       async def check_filelist(self):
                # XXX need to remove data from disabled mirrors
                if not self.enabled:
                        return
@@ -291,7 +285,7 @@ class Mirror(Object):
                http = tornado.httpclient.AsyncHTTPClient()
 
                try:
-                       response = yield http.fetch(self.url + ".filelist",
+                       response = await http.fetch(self.url + ".filelist",
                                headers={ "Pragma" : "no-cache" })
                except tornado.httpclient.HTTPError as e:
                        logging.error("Error getting filelist from %s: %s" % (self.hostname, e))
index 6b37e23cafde5ce26552698c6bbae0b9c885233c..b02b992151db3e1672570e17218b8b868cd15da2 100644 (file)
@@ -4,7 +4,6 @@ import hashlib
 import logging
 import os
 import re
-import tornado.gen
 import urllib.parse
 import yabencode
 
@@ -544,8 +543,7 @@ class Releases(Object):
 
                return file
 
-       @tornado.gen.coroutine
-       def scan_files(self, basepath="/pub/mirror"):
+       async def scan_files(self, basepath="/pub/mirror"):
                for release in self:
                        logging.debug("Scanning %s..." % release)
 
index 543fcdf25f1adb35026ec40b389f5fa758817ea1..e81fd29201bd32f1fa469c70d3fe249b50b536be 100644 (file)
@@ -2,14 +2,12 @@
 
 import datetime
 import logging
-import tornado.gen
 import twython
 
 from .misc import Object
 
 class Tweets(Object):
-       @tornado.gen.coroutine
-       def tweet(self):
+       async def tweet(self):
                """
                        Sends a random promotional tweet
                """
index ce1e4e0d7e79ecdeb54d9580c06d212df653c53e..6c2539de2ccdf1e096b7d33e92dcdc7f5dc2a50b 100644 (file)
@@ -4,7 +4,6 @@ import difflib
 import logging
 import os.path
 import re
-import tornado.gen
 import urllib.parse
 
 from . import misc
index 158d611349fc816f6ee0331f71f7e1b2c54909c5..fef5e00b1fa85abd0bf00ab549a59f84dad30d0b 100644 (file)
@@ -4,7 +4,6 @@ import hashlib
 import hmac
 import json
 import tornado.httpclient
-import tornado.gen
 import urllib.parse
 
 from .misc import Object
@@ -37,8 +36,7 @@ class ZeiterfassungClient(Object):
 
                return h.hexdigest()
 
-       @tornado.gen.coroutine
-       def send_request(self, path, **kwargs):
+       async def send_request(self, path, **kwargs):
                url = urllib.parse.urljoin(self.url, path)
 
                # Query arguments are all keyword arguments
@@ -56,9 +54,7 @@ class ZeiterfassungClient(Object):
                )
 
                # Send the request
-               response = yield self.backend.http_client.fetch(request)
+               response = await self.backend.http_client.fetch(request)
 
                # Decode the JSON response
-               d = json.loads(response.body.decode())
-
-               raise tornado.gen.Return(d)
+               return json.loads(response.body.decode())
index 25a0fb92819873c3d93a78b4b8a25c94a707e3a6..f74e0816c07624d60cafdcd786772808fc922898 100644 (file)
@@ -1,7 +1,6 @@
 #!@PYTHON@
 
 import sys
-import tornado.gen
 import tornado.ioloop
 import tornado.options
 
@@ -20,9 +19,8 @@ class TaskRunner(object):
                        arguments asynchronically and exits the program in
                        case on a non-zero exit code
                """
-               @tornado.gen.coroutine
-               def task():
-                       yield self.backend.run_task(name, *args, **kwargs)
+               async def task():
+                       await self.backend.run_task(name, *args, **kwargs)
 
                return self.ioloop.run_sync(task)
 
index 4bd563ffec66ec11f5ca20cd0ac53aec3d69f156..e20a553c42bfec58c1826d04800a90fa63cd18f2 100644 (file)
@@ -92,9 +92,8 @@ class RegisterHandler(base.BaseHandler):
 
                self.render("auth/register.html")
 
-       @tornado.gen.coroutine
        @base.ratelimit(minutes=24*60, requests=5)
-       def post(self):
+       async def post(self):
                uid   = self.get_argument("uid")
                email = self.get_argument("email")
 
@@ -102,7 +101,7 @@ class RegisterHandler(base.BaseHandler):
                last_name  = self.get_argument("last_name")
 
                # Check if this is a spam account
-               is_spam = yield self.backend.accounts.check_spam(uid, email,
+               is_spam = await self.backend.accounts.check_spam(uid, email,
                        address=self.get_remote_ip())
 
                if is_spam:
index 437050acb9126c70376cbb24b0fa7efb4c1c5395..e000ca33718516c59b5d827e4bdd1fb532c4837a 100644 (file)
@@ -14,11 +14,10 @@ from ..decorators import *
 from .. import util
 
 def blacklisted(method):
-       @tornado.gen.coroutine
        @functools.wraps(method)
-       def wrapper(self, *args, **kwargs):
+       async def wrapper(self, *args, **kwargs):
                # Check if remote is blacklisted
-               is_blacklisted = yield self.remote.is_blacklisted()
+               is_blacklisted = await self.remote.is_blacklisted()
 
                # If so, redirect to the blocked page
                if is_blacklisted:
index 2429cdb7de266d759947e18467b6a04248b4a178..b22b8cad5ffba49b730cd09b763a35a83ce8a076 100644 (file)
@@ -1,7 +1,6 @@
 #!/usr/bin/python3
 
 import iso3166
-import tornado.gen
 import tornado.web
 
 from . import base
@@ -43,9 +42,8 @@ class DonateHandler(base.BaseHandler):
                        country=country, first_name=first_name, last_name=last_name,
                        amount=amount, currency=currency, frequency=frequency)
 
-       @tornado.gen.coroutine
        @base.ratelimit(minutes=24*60, requests=5)
-       def post(self):
+       async def post(self):
                amount    = self.get_argument("amount")
                currency  = self.get_argument("currency", "EUR")
                frequency = self.get_argument("frequency")
@@ -82,7 +80,7 @@ class DonateHandler(base.BaseHandler):
 
                # Send request to Zeiterfassung
                try:
-                       response = yield self.backend.zeiterfassung.send_request(
+                       response = await self.backend.zeiterfassung.send_request(
                                "/api/v1/donations/create/ipfire-project", **args)
 
                except Exception:
index f15718bfa1ee7eab629004d3bedb15436fa705a9..5807542a8fa0550392adb3cbd7eb27f05c17c986 100644 (file)
@@ -2,7 +2,6 @@
 #!/usr/bin/python
 
 import logging
-import tornado.gen
 import tornado.web
 
 from . import base
@@ -13,8 +12,7 @@ class IndexHandler(base.BaseHandler):
 
 
 class LookupHandler(base.BaseHandler):
-       @tornado.gen.coroutine
-       def get(self, address):
+       async def get(self, address):
                peer = self.geoip.get_all(address)
                if peer:
                        peer["country_name"] = self.geoip.get_country_name(peer.country)
@@ -23,22 +21,21 @@ class LookupHandler(base.BaseHandler):
                address = self.geoip.lookup(address)
 
                # Lookup blacklists
-               is_blacklisted = yield address.is_blacklisted()
+               is_blacklisted = await address.is_blacklisted()
 
                self.render("location/lookup.html",
                        address=address, is_blacklisted=is_blacklisted, peer=peer)
 
 
 class BlacklistsHandler(base.BaseHandler):
-       @tornado.gen.coroutine
-       def get(self, address):
+       async def get(self, address):
                peer = self.geoip.get_all(address)
 
                # Lookup address
                address = self.geoip.lookup(address)
 
                # Lookup blacklists
-               blacklists = yield address.get_blacklists()
+               blacklists = await address.get_blacklists()
 
                self.render("location/blacklists.html",
                        address=address, blacklists=blacklists)
index abf31ce0f314b65399446ad8db86ab761ca3107b..c08478043451c7344c817bb426c03a4739e78b5c 100644 (file)
@@ -2,7 +2,6 @@
 
 import email.mime.text
 import logging
-import tornado.gen
 import tornado.process
 
 from . import base
@@ -13,7 +12,6 @@ class SubscribeHandler(base.BaseHandler):
                # Makes sure that we call blacklist for everything
                pass
 
-       @tornado.gen.coroutine
        @base.ratelimit(minutes=15, requests=5)
        def post(self):
                address = self.get_argument("email")