From: Flole Date: Fri, 12 Jun 2026 13:39:39 +0000 (+0000) Subject: Fix pcloud caching X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=bb09b4faed80eb4ddc20da945d397b9f36e49fdd;p=thirdparty%2Ftvheadend.git Fix pcloud caching --- diff --git a/.github/workflows/build-ci.yml b/.github/workflows/build-ci.yml index f278d236f..dc11024d1 100644 --- a/.github/workflows/build-ci.yml +++ b/.github/workflows/build-ci.yml @@ -17,6 +17,8 @@ jobs: runs-on: ubuntu-latest continue-on-error: true name: Build on Raspberry Pi ${{ matrix.arch }} + env: + PCLOUD_TOKEN: ${{ secrets.PCLOUD_TOKEN }} strategy: matrix: arch: [armv6l, armv7l, aarch64] @@ -102,6 +104,7 @@ jobs: # Pass some environment variables to the container env: | # YAML, but pipe character is necessary artifact_name: git-${{ matrix.distro }}_${{ matrix.arch }} + PCLOUD_TOKEN: ${{ secrets.PCLOUD_TOKEN }} # The shell to run commands with in the container @@ -159,6 +162,8 @@ jobs: runs-on: ubuntu-latest continue-on-error: true name: Build on native ${{ matrix.container[1] }} + env: + PCLOUD_TOKEN: ${{ secrets.PCLOUD_TOKEN }} strategy: matrix: container: [["i386/ubuntu:trusty", "i386-ubuntu-trusty"], ["ubuntu:trusty", "ubuntu-trusty"], ["i386/ubuntu:xenial", "i386-ubuntu-xenial"], ["ubuntu:xenial", "ubuntu-xenial"], ["ubuntu:bionic", "ubuntu-bionic"], ["ubuntu:focal", "ubuntu-focal"], ["ubuntu:jammy", "ubuntu-jammy"], ["ubuntu:noble", "ubuntu-noble"], ["ubuntu:plucky", "ubuntu-plucky"], ["ubuntu:questing", "ubuntu-questing"], ["ubuntu:resolute", "ubuntu-resolute"], ["i386/debian:stretch", "i386-debian-stretch"], ["debian:stretch", "debian-stretch"], ["i386/debian:buster", "i386-debian-buster"], ["debian:buster", "debian-buster"], ["i386/debian:bullseye", "i386-debian-bullseye"], ["debian:bullseye", "debian-bullseye"], ["i386/debian:bookworm", "i386-debian-bookworm"], ["debian:bookworm", "debian-bookworm"], ["i386/debian:trixie", "i386-debian-trixie"], ["debian:trixie", "debian-trixie"], ["i386/debian:sid", "i386-debian-sid"], ["debian:sid", "debian-sid"]] @@ -199,7 +204,7 @@ jobs: SCRIPT: | cd /workspace AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\ --enable-ffmpeg_static\ --enable-hdhomerun_static\ --python=python3 ./Autobuild.sh ${{ (startsWith(matrix.container[0], 'i386') && '-a i386') || '' }} - run: docker exec build-container bash -c "$SCRIPT" + run: docker exec -e PCLOUD_TOKEN build-container bash -c "$SCRIPT" - name: copy-result env: SCRIPT: | @@ -221,6 +226,8 @@ jobs: runs-on: ubuntu-latest continue-on-error: true name: Build on native fedora:${{ matrix.releasever }} + env: + PCLOUD_TOKEN: ${{ secrets.PCLOUD_TOKEN }} strategy: matrix: releasever: ["37", "38", "39", "40", "41", "42", "43", "44", "rawhide"] @@ -260,6 +267,8 @@ jobs: releasever: ["9", "10"] runner: ["ubuntu-latest", "ubuntu-24.04-arm"] runs-on: ${{ matrix.runner }} + env: + PCLOUD_TOKEN: ${{ secrets.PCLOUD_TOKEN }} container: image: "almalinux:${{ matrix.releasever }}" steps: diff --git a/.github/workflows/build-cloudsmith.yml b/.github/workflows/build-cloudsmith.yml index f13ff81b2..16f77aa11 100644 --- a/.github/workflows/build-cloudsmith.yml +++ b/.github/workflows/build-cloudsmith.yml @@ -18,6 +18,7 @@ jobs: CLOUDSMITH_REPO: ${{ vars.CLOUDSMITH_REPO }} CLOUDSMITH_OWNER: ${{ secrets.CLOUDSMITH_OWNER }} CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_KEY }} + PCLOUD_TOKEN: ${{ secrets.PCLOUD_TOKEN }} NODIRTY: ${{ secrets.NODIRTY }} strategy: matrix: @@ -106,6 +107,7 @@ jobs: CLOUDSMITH_REPO: ${{ vars.CLOUDSMITH_REPO }} CLOUDSMITH_OWNER: ${{ secrets.CLOUDSMITH_OWNER }} CLOUDSMITH_API_KEY: ${{ secrets.CLOUDSMITH_KEY }} + PCLOUD_TOKEN: ${{ secrets.PCLOUD_TOKEN }} NODIRTY: ${{ secrets.NODIRTY }} @@ -164,6 +166,8 @@ jobs: runs-on: ubuntu-latest continue-on-error: true name: Build on native ${{ matrix.container[1] }} + env: + PCLOUD_TOKEN: ${{ secrets.PCLOUD_TOKEN }} strategy: matrix: container: [["i386/ubuntu:trusty", "i386-ubuntu-trusty"], ["ubuntu:trusty", "ubuntu-trusty"], ["i386/ubuntu:xenial", "i386-ubuntu-xenial"], ["ubuntu:xenial", "ubuntu-xenial"], ["ubuntu:bionic", "ubuntu-bionic"], ["ubuntu:focal", "ubuntu-focal"], ["ubuntu:jammy", "ubuntu-jammy"], ["ubuntu:noble", "ubuntu-noble"], ["ubuntu:plucky", "ubuntu-plucky"], ["ubuntu:questing", "ubuntu-questing"], ["ubuntu:resolute", "ubuntu-resolute"], ["i386/debian:stretch", "i386-debian-stretch"], ["debian:stretch", "debian-stretch"], ["i386/debian:buster", "i386-debian-buster"], ["debian:buster", "debian-buster"], ["i386/debian:bullseye", "i386-debian-bullseye"], ["debian:bullseye", "debian-bullseye"], ["i386/debian:bookworm", "i386-debian-bookworm"], ["debian:bookworm", "debian-bookworm"], ["i386/debian:trixie", "i386-debian-trixie"], ["debian:trixie", "debian-trixie"], ["i386/debian:sid", "i386-debian-sid"], ["debian:sid", "debian-sid"]] @@ -204,7 +208,7 @@ jobs: SCRIPT: | cd /workspace AUTOBUILD_CONFIGURE_EXTRA=--enable-ccache\ --enable-ffmpeg_static\ --enable-hdhomerun_static\ --python=python3 ./Autobuild.sh ${{ (startsWith(matrix.container[0], 'i386') && '-a i386') || '' }} - run: docker exec build-container bash -c "$SCRIPT" + run: docker exec -e PCLOUD_TOKEN build-container bash -c "$SCRIPT" - name: copy-result env: SCRIPT: | @@ -230,6 +234,8 @@ jobs: runs-on: ubuntu-latest continue-on-error: true name: Build on native fedora:${{ matrix.releasever }} + env: + PCLOUD_TOKEN: ${{ secrets.PCLOUD_TOKEN }} strategy: matrix: releasever: ["37", "38", "39", "40", "41", "42", "43", "44", "rawhide"] @@ -274,6 +280,8 @@ jobs: releasever: ["9", "10"] runner: ["ubuntu-latest", "ubuntu-24.04-arm"] runs-on: ${{ matrix.runner }} + env: + PCLOUD_TOKEN: ${{ secrets.PCLOUD_TOKEN }} container: image: "almalinux:${{ matrix.releasever }}" steps: diff --git a/Makefile.static b/Makefile.static index e6cd33d98..7bdaac630 100644 --- a/Makefile.static +++ b/Makefile.static @@ -34,8 +34,7 @@ # Optional inputs # # PCLOUD_CACHE - Use cached builds from pcloud -# PCLOUD_USER - The pcloud user account for uploads -# PCLOUD_PASS - The pcloud password +# PCLOUD_TOKEN - The pcloud access token # PCLOUD_BASEDIR - The base directory for uploads # PCLOUD_HASHDIR - The public hash for the base directory for downloads # @@ -54,8 +53,7 @@ export ROOTDIR export BUILDDIR export LIBDIR export PCLOUD_CACHE -export PCLOUD_USER -export PCLOUD_PASS +export PCLOUD_TOKEN export PCLOUD_BASEDIR export PCLOUD_HASHDIR @@ -79,7 +77,7 @@ endif # Download and Upload on demand .PHONY: libcacheput libcacheput: build -ifneq ($(PCLOUD_USER),) +ifneq ($(PCLOUD_TOKEN),) @$(ROOTDIR)/support/lib.sh upload $(LIB_NAME) $(LIB_FILES) endif diff --git a/support/lib.sh b/support/lib.sh index 3921685bd..1d6e3e0cd 100755 --- a/support/lib.sh +++ b/support/lib.sh @@ -158,7 +158,7 @@ function upload P="${BUILDDIR}/.${LIB_NAME}-${LIB_HASH}.tgz" # Can't upload - [ -z "${PCLOUD_USER}" -o -z "${PCLOUD_PASS}" ] && return 0 + [ -z "${PCLOUD_TOKEN}" ] && return 0 # Don't need to upload [ -f "${P}" ] && return 0 diff --git a/support/pcloud.py b/support/pcloud.py index 6adde58bc..f4787b4a7 100755 --- a/support/pcloud.py +++ b/support/pcloud.py @@ -9,10 +9,8 @@ import os import sys -import traceback import requests import json -from hashlib import sha1 from io import BytesIO from os.path import basename @@ -20,18 +18,14 @@ def env(key): if key in os.environ: return os.environ[key] return None -PCLOUD_USER=env('PCLOUD_USER') -PCLOUD_PASS=env('PCLOUD_PASS') -PCLOUD_CA_CERTS=env('PCLOUD_CA_CERTS') or os.path.dirname(os.path.realpath(__file__)) + '/pcloud-ca-bundle.crt' +# FIX: Switched from USER/PASS to OAuth 2.0 Bearer Token +PCLOUD_TOKEN = env('PCLOUD_TOKEN') -DEBUG=False +# US/Default accounts use api.pcloud.com. EU accounts MUST use eapi.pcloud.com. +PCLOUD_ENDPOINT = env('PCLOUD_ENDPOINT') or 'https://api.pcloud.com/' +PCLOUD_CA_CERTS = env('PCLOUD_CA_CERTS') or os.path.dirname(os.path.realpath(__file__)) + '/pcloud-ca-bundle.crt' -# File open flags https://docs.pcloud.com/methods/fileops/file_open.html -O_WRITE = int('0x0002', 16) -O_CREAT = int('0x0040', 16) -O_EXCL = int('0x0080', 16) -O_TRUNC = int('0x0200', 16) -O_APPEND = int('0x0400', 16) +DEBUG = False def error(lvl, msg, *args): sys.stderr.write(msg % args + '\n') @@ -57,44 +51,27 @@ def pcloud_normpath(path): path = '/' + path return path -def pcloud_extract_publink_data(text): - text = text.decode('utf-8') - pos = text.find('var publinkData = {') - if pos < 0: raise(ContentsError) - text = text[pos+18:] - pos = text.find('};') - if pos < 0: raise(ContentsError) - text = text[:pos+1] - return json.loads(text) - -# Exceptions class AuthenticationError(Exception): """ Authentication failed """ -# Exceptions class ContentsError(Exception): - """ Authentication failed """ + """ Contents Error """ -# Validation class RequiredParameterCheck(object): - """ A decorator that checks function parameter - """ - def __init__(self, required): self.required = required def __call__(self, func): def wrapper(*args, **kwargs): - found_paramater = False + found_parameter = False for req in self.required: if req in kwargs: - found_paramater = True + found_parameter = True break - if found_paramater: + if found_parameter: return func(*args, **kwargs) else: - raise ValueError('One required parameter `%s` is missing', - ', '.join(self.required)) + raise ValueError('One required parameter `%s` is missing' % ', '.join(self.required)) wrapper.__name__ = func.__name__ wrapper.__dict__.update(func.__dict__) wrapper.__doc__ = func.__doc__ @@ -102,55 +79,29 @@ class RequiredParameterCheck(object): class PyCloud(object): - endpoint = 'https://api.pcloud.com/' - - def __init__(self, username, password): - self.username = username.lower().encode('utf-8') - self.password = password.encode('utf-8') + def __init__(self, oauth_token): + self.endpoint = PCLOUD_ENDPOINT + self.oauth_token = oauth_token self.session = requests.Session() - self.auth_token = self.get_auth_token() + self._verify_token() + + def _verify_token(self): + resp = self._do_request('userinfo') + if 'email' not in resp: + raise AuthenticationError("OAuth Authentication failed. Check your PCLOUD_TOKEN or PCLOUD_ENDPOINT region.") def _do_request(self, method, authenticate=True, json=True, **kw): + params = kw.copy() + # Pass the OAuth token dynamically to API methods if authenticate: - params = {'auth': self.auth_token} - else: - params = {} - params.update(kw) - #log.debug('Doing request to %s%s', self.endpoint, method) - #log.debug('Params: %s', params) + params['access_token'] = self.oauth_token + resp = self.session.get(self.endpoint + method, params=params, timeout=30, verify=PCLOUD_CA_CERTS) if json: return resp.json() else: return resp.content - # Authentication - def getdigest(self): - resp = self._do_request('getdigest', authenticate=False) - try: - return bytes(resp['digest'], 'utf-8') - except: - return bytes(resp['digest']) - - def get_auth_token(self): - digest = self.getdigest() - try: - uhash = bytes(sha1(self.username).hexdigest(), 'utf-8') - except: - uhash = bytes(sha1(self.username).hexdigest()) - passworddigest = sha1(self.password + uhash + digest) - params = { - 'getauth': 1, - 'logout': 1, - 'username': self.username.decode('utf-8'), - 'digest': digest.decode('utf-8'), - 'passworddigest': passworddigest.hexdigest()} - resp = self._do_request('userinfo', authenticate=False, **params) - if 'auth' not in resp: - raise(AuthenticationError) - return resp['auth'] - - # Folders @RequiredParameterCheck(('path', 'folderid')) def createfolder(self, **kwargs): return self._do_request('createfolder', **kwargs) @@ -172,22 +123,13 @@ class PyCloud(object): return self._do_request('deletefolderrecursive', **kwargs) def _upload(self, method, files, **kwargs): - kwargs['auth'] = self.auth_token - resp = self.session.post(self.endpoint + method, files=files, data=kwargs, verify=PCLOUD_CA_CERTS) + data = kwargs.copy() + data['access_token'] = self.oauth_token + resp = self.session.post(self.endpoint + method, files=files, data=data, verify=PCLOUD_CA_CERTS) return resp.json() @RequiredParameterCheck(('files', 'data')) def uploadfile(self, **kwargs): - """ upload a file to pCloud - - 1) You can specify a list of filenames to read - files=[('/home/pcloud/foo.txt', 'foo-on-cloud.txt'), - ('/home/pcloud/bar.txt', 'bar-on-cloud.txt')] - - 2) you can specify binary data via the data parameter and - need to specify the filename too - data='Hello pCloud', filename='foo.txt' - """ if 'files' in kwargs: files = {} upload_files = kwargs.pop('files') @@ -195,7 +137,7 @@ class PyCloud(object): filename = basename(f[1]) files = {filename: (filename, open(f[0], 'rb'))} kwargs['filename'] = filename - else: # 'data' in kwargs: + else: files = {'f': (kwargs.pop('filename'), kwargs.pop('data'))} return self._upload('uploadfile', files, **kwargs) @@ -207,9 +149,6 @@ class PyCloud(object): def downloadfile(self, **kwargs): return self._do_request('downloadfile', **kwargs) - def copyfile(self, **kwargs): - pass - @RequiredParameterCheck(('path', 'fileid')) def checksumfile(self, **kwargs): return self._do_request('checksumfile', **kwargs) @@ -221,94 +160,10 @@ class PyCloud(object): def renamefile(self, **kwargs): return self._do_request('renamefile', **kwargs) - # Auth API methods - def sendverificationemail(self, **kwargs): - return self._do_request('sendverificationemail', **kwargs) - - def verifyemail(self, **kwargs): - return self._do_request('verifyemail', **kwargs) - - def changepassword(self, **kwargs): - return self._do_request('changepassword', **kwargs) - - def lostpassword(self, **kwargs): - return self._do_request('lostpassword', **kwargs) - - def resetpassword(self, **kwargs): - return self._do_request('resetpassword', **kwargs) - - def register(self, **kwargs): - return self._do_request('register', **kwargs) - - def invite(self, **kwargs): - return self._do_request('invite', **kwargs) - - def userinvites(self, **kwargs): - return self._do_request('userinvites', **kwargs) - - def logout(self, **kwargs): - return self._do_request('logout', **kwargs) - - def listtokens(self, **kwargs): - return self._do_request('listtokens', **kwargs) - - def deletetoken(self, **kwargs): - return self._do_request('deletetoken', **kwargs) - - # File API methods - @RequiredParameterCheck(('flags',)) - def file_open(self, **kwargs): - return self._do_request('file_open', **kwargs) - - @RequiredParameterCheck(('fd',)) - def file_read(self, **kwargs): - return self._do_request('file_read', json=False, **kwargs) - - @RequiredParameterCheck(('fd',)) - def file_pread(self, **kwargs): - return self._do_request('file_pread', json=False, **kwargs) - - @RequiredParameterCheck(('fd', 'data')) - def file_pread_ifmod(self, **kwargs): - return self._do_request('file_pread_ifmod', json=False, **kwargs) - - @RequiredParameterCheck(('fd',)) - def file_size(self, **kwargs): - return self._do_request('file_size', **kwargs) - - @RequiredParameterCheck(('fd',)) - def file_truncate(self, **kwargs): - return self._do_request('file_truncate', **kwargs) - - @RequiredParameterCheck(('fd', 'data')) - def file_write(self, **kwargs): - files = {'filename': BytesIO(kwargs['data'])} - return self._upload('file_write', files, **kwargs) - - @RequiredParameterCheck(('fd',)) - def file_pwrite(self, **kwargs): - return self._do_request('file_pwrite', **kwargs) - - @RequiredParameterCheck(('fd',)) - def file_checksum(self, **kwargs): - return self._do_request('file_checksum', **kwargs) - - @RequiredParameterCheck(('fd',)) - def file_seek(self, **kwargs): - return self._do_request('file_seek', **kwargs) - - @RequiredParameterCheck(('fd',)) - def file_close(self, **kwargs): - return self._do_request('file_close', **kwargs) - - @RequiredParameterCheck(('fd',)) - def file_lock(self, **kwargs): - return self._do_request('file_lock', **kwargs) - def simple(method, **kwargs): - if not PCLOUD_USER or not PCLOUD_PASS: - error(2, 'No credentals') - pcloud = PyCloud(PCLOUD_USER, PCLOUD_PASS) + if not PCLOUD_TOKEN: + error(2, 'No OAuth credentials provided. Ensure PCLOUD_TOKEN is set.') + pcloud = PyCloud(PCLOUD_TOKEN) return getattr(pcloud, method)(**kwargs) def do_listfolder(*args): @@ -326,9 +181,9 @@ def do_createfolder(*args): if len(args) < 1: error(1, 'createfolder [path]') path = pcloud_normpath(args[0]) r = simple('createfolder', path=path) - if r['result'] == 2004: # Folder already exists + if r.get('result') == 2004: return 0 - if r['result']: + if r.get('result'): error(10, 'Unable to create folder %s (%s: %s)', path, r['result'], r['error']) def do_upload(*args): @@ -336,7 +191,7 @@ def do_upload(*args): path = pcloud_normpath(args[0]) path, file = os.path.split(path) r = simple('uploadfile', path=path, files=[(args[1], file)], nopartial=1) - if r['result'] == 2005: # directory does not exist + if r.get('result') == 2005: s = split_path(path) s.reverse() p = '' @@ -345,29 +200,35 @@ def do_upload(*args): if p != '//': do_createfolder(p) else: - p = '/' + p = '' r = simple('uploadfile', path=path, files=[(args[1], file)], nopartial=1) - if r['result']: + if r.get('result'): error(10, 'Unable to upload %s to %s (%s: %s)', args[1], path, r['result'], r['error']) def do_publink_download(*args): if len(args) < 3: error(1, 'download [root-hash] [full path] [output path]') session = requests.Session() path = pcloud_normpath(args[1]) - resp = session.get('https://my.pcloud.com/publink/show?code=%s' % args[0], timeout=30, verify=PCLOUD_CA_CERTS) + + resp = session.get(PCLOUD_ENDPOINT + 'showpublink?code=%s' % args[0], timeout=30, verify=PCLOUD_CA_CERTS) if resp.status_code != 200: - error(10, 'Unable to retreive publink %s', args[0]) - pdata = pcloud_extract_publink_data(resp.content) - meta = pdata['metadata'] - if not meta: - error(10, 'No metadata, object probably does not exist!') + error(10, 'Unable to retrieve publink %s', args[0]) + + j = resp.json() + if j.get('result') != 0 or 'metadata' not in j: + error(10, 'No metadata found, object probably does not exist or link is invalid!') + + meta = j['metadata'] s = split_path(path[1:]) s.reverse() name = s.pop() + if meta['name'] != name: error(10, 'Root folder name does not match ("%s" - "%s")', meta['name'], name) - ctx = meta['contents'] + + ctx = meta.get('contents', []) fctx = None + while s: name = s.pop() found = 0 @@ -382,23 +243,33 @@ def do_publink_download(*args): break if not found: error(10, 'Folder name "%s" not found', name) + if not fctx: error(10, 'Filename "%s" not found', path) - resp = session.get('https://api.pcloud.com/getpublinkdownload?fileid=%s&hashCache=%s&code=%s' % (fctx['fileid'], fctx['hash'], args[0]), timeout=30, verify=PCLOUD_CA_CERTS) + + resp = session.get(PCLOUD_ENDPOINT + 'getpublinkdownload?fileid=%s&code=%s' % (fctx['fileid'], args[0]), timeout=30, verify=PCLOUD_CA_CERTS) if resp.status_code != 200: error(10, 'Unable to get file json for "%s"!' % path) - j = resp.json() - if len(j['hosts']) <= 0: - error(10, 'No hosts?') - for idx in range(len(j['hosts'])): - resp = session.get('https://%s%s' % (j['hosts'][idx], j['path']), timeout=30, verify=PCLOUD_CA_CERTS) + + dl_j = resp.json() + if dl_j.get('result') != 0: + error(10, 'Error obtaining download link: %s', dl_j.get('error')) + + if len(dl_j.get('hosts', [])) <= 0: + error(10, 'No download hosts returned by API') + + for idx in range(len(dl_j['hosts'])): + resp = session.get('https://%s%s' % (dl_j['hosts'][idx], dl_j['path']), timeout=30, verify=PCLOUD_CA_CERTS) if resp.status_code == 200: break + if resp.status_code != 200: - error(10, 'Unable to retreive file content for "%s"!' % path) + error(10, 'Unable to retrieve file content for "%s"!' % path) + if len(resp.content) == 0: - error(10, 'Empty') - fp = open(args[2], sys.version_info[0] < 3 and "w+" or "bw+") + error(10, 'Downloaded file is empty') + + fp = open(args[2], sys.version_info[0] < 3 and "w+" or "wb+") fp.write(resp.content) fp.close() @@ -421,4 +292,4 @@ def main(argv): do_unknown() if __name__ == "__main__": - main(sys.argv) + main(sys.argv) \ No newline at end of file