api: Move the file streaming into the backend

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/api/debuginfo.py b/src/api/debuginfo.py
index 8080c7840398c1817122bb2c13823d49f7914c5c..708027cc35003ed285fa7027c41fe550486a2087 100644 (file)
--- a/src/api/debuginfo.py
+++ b/src/api/debuginfo.py
@@ -18,7 +18,6 @@
 #                                                                             #
 ###############################################################################
 
-import asyncio
 import fastapi
 
 from . import app
@@ -38,17 +37,6 @@ async def get(buildid: str) -> fastapi.responses.StreamingResponse:
        if not file:
                raise fastapi.HTTPException(404, "Could not find debuginfo in %s" % package)
 
-       # Create a helper function that will stream the file chunk by chunk
-       async def stream():
-               f = await file.open()
-
-               while True:
-                       chunk = await asyncio.to_thread(f.read, 128 * 1024)
-                       if not chunk:
-                               break
-
-                       yield chunk
-
        # Stream the payload
        return fastapi.responses.StreamingResponse(
-               stream(), media_type="application/octet-stream")
+               file.stream(), media_type="application/octet-stream")

diff --git a/src/api/packages.py b/src/api/packages.py
index 8e75f1dd10749da0d842b443d7d24daefc4bc888..a55c2082079f3648912ddfe089851eadca0da482 100644 (file)
--- a/src/api/packages.py
+++ b/src/api/packages.py
@@ -18,7 +18,6 @@
 #                                                                             #
 ###############################################################################
 
-import asyncio
 import fastapi
 import pydantic
 
@@ -91,6 +90,8 @@ async def get_filelist_by_uuid(
        return [file async for file in await package.get_files()]
 
 
+# XXX This endpoint need some ratelimiting applied
+
 @app.get("/packages/{uuid:uuid}/download/{path:path}", include_in_schema=False)
 async def download_file(
        path: str,
@@ -122,19 +123,8 @@ async def download_file(
                "X-Robots-Tag"        : "noindex",
        }
 
-       # Create a helper function that will stream the file chunk by chunk
-       async def stream():
-               f = await file.open()
-
-               while True:
-                       chunk = await asyncio.to_thread(f.read, 128 * 1024)
-                       if not chunk:
-                               break
-
-                       yield chunk
-
        return fastapi.responses.StreamingResponse(
-               stream(), media_type=file.mimetype, headers=headers,
+               file.stream(), media_type=file.mimetype, headers=headers,
        )
 
 

diff --git a/src/buildservice/packages.py b/src/buildservice/packages.py
index 4568b424a8c3e66aaee52fd68519ee1642ae5bee..cf4c9cbbc4a7e69802afcca34c0a53678c0f3e0e 100644 (file)
--- a/src/buildservice/packages.py
+++ b/src/buildservice/packages.py
@@ -780,6 +780,22 @@ class File(sqlmodel.SQLModel, table=True):
                # Read the payload in a separate thread
                return await asyncio.to_thread(func)
 
+       async def stream(self, chunk_size=128 * 1024):
+               """
+                       A helper function to stream the payload chunk by chunk
+               """
+               # Open the file
+               f = await self.open()
+
+               # Read a chunk of up to chunk size bytes and return it
+               # until we have read everything.
+               while True:
+                       chunk = await asyncio.to_thread(f.read, chunk_size)
+                       if not chunk:
+                               break
+
+                       yield chunk
+
        async def sendfile(self, dst, chunk_size=65536):
                """
                        Sends the payload of the file into the given file descriptor