From 268f68aeecc15a79c4efca0b2942bb86884266fa Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sat, 7 Jun 2025 17:36:51 +0200 Subject: [PATCH 01/57] btrfs/blake2 --- sh/fs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sh/fs.sh b/sh/fs.sh index ac46f6d..0db5696 100644 --- a/sh/fs.sh +++ b/sh/fs.sh @@ -5,7 +5,7 @@ rwx_fs_make_btrfs() { if [ -b "${device}" ]; then set -- \ --force \ - --checksum "sha256" + --checksum "blake2" if [ -n "${label}" ]; then set -- "${@}" \ --label "${label}" From a167190d6c0a939465270b9dc5f34ff5d65ba1ec Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 23 Feb 2025 19:27:17 +0100 Subject: [PATCH 02/57] lint --- sh/python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sh/python.sh b/sh/python.sh index 44ea181..59773c7 100644 --- a/sh/python.sh +++ b/sh/python.sh @@ -9,6 +9,6 @@ rwx_python_venv() { local path="${1}" [ -d "${path}" ] || return 1 - export VIRTUAL_ENV="${path}" && \ + export VIRTUAL_ENV="${path}" && export PATH="${VIRTUAL_ENV}/bin:${PATH}" } From 02cf8f83e7e5dbc0b7556c8982993baf25dc65a4 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 16 Mar 2025 01:01:52 +0100 Subject: [PATCH 03/57] cp --- rwx/sw/youtube/__init__.py | 142 +++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 rwx/sw/youtube/__init__.py diff --git a/rwx/sw/youtube/__init__.py b/rwx/sw/youtube/__init__.py new file mode 100644 index 0000000..06f5bdc --- /dev/null +++ b/rwx/sw/youtube/__init__.py @@ -0,0 +1,142 @@ +import yt_dlp + +# playlists +# … +# entries + +# playlists / entries +# title +# id + +# playlist +# entries + +# playlist / entries +# … + + +# videos +# id +# channel +# channel_id +# title +# channel_follower_count +# description +# tags +# thumbnails +# uploader_id +# uploader +# entries + +# videos / entries +# id +# title +# description truncated +# duration +# thumbnails +# view_count + + +# video +# id +# title +# formats +# thumbnails +# thumbnail +# description +# +# duration +# view_count +# categories +# tags +# fulltitle + + +def extract_playlist(playlist_id): + options = { + "quiet": False, + "extract_flat": True, + "force_generic_extractor": False, + "ignoreerrors": False, + } + url = f"https://youtube.com/playlist?list={playlist_id}" + print(f"""\ +{options} + +{url} + +""") + with yt_dlp.YoutubeDL(options) as y: + return y.extract_info(url, download=False) + + +def extract_playlists(channel_id): + options = { + "quiet": False, + "extract_flat": True, + "force_generic_extractor": False, + "ignoreerrors": False, + "skip_download": True, + } + url = f"https://youtube.com/channel/{channel_id}/playlists" + print(f"""\ +{options} + +{url} + +""") + with yt_dlp.YoutubeDL(options) as y: + return y.extract_info(url, download=False) + + +def download_video(video_id): + options = { + "format": "bestvideo[ext=mp4]+bestaudio[ext=mp4]", + "outtmpl": "%(id)s.%(ext)s", + "writesubtitles": True, + "writethumbnail": True, + "ignoreerrors": False, + } + url = f"https://youtu.be/{video_id}" + print(f"""\ +{options} + +{url} + +""") + with yt_dlp.YoutubeDL(options) as y: + return y.download([url]) + + +def extract_video(video_id): + options = { + "quiet": False, + "ignoreerrors": False, + } + url = f"https://youtu.be/{video_id}" + print(f"""\ +{options} + +{url} + +""") + with yt_dlp.YoutubeDL(options) as y: + return y.extract_info(url, download=False) + + +def extract_videos(channel_id): + options = { + "quiet": False, + "extract_flat": True, + "force_generic_extractor": False, + "ignoreerrors": False, + } + url = f"https://youtube.com/channel/{channel_id}/videos" + print(f"""\ +{options} + +{url} + +""") + with yt_dlp.YoutubeDL(options) as y: + return y.extract_info(url, download=False) From 0b5d96116c7e32fe256f38562ff8cbdf439717f8 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 16 Mar 2025 15:52:43 +0100 Subject: [PATCH 04/57] wip --- rwx/sw/youtube/__init__.py | 193 ++++++++++++++++++++++--------------- rwx/sw/yt_dlp/__init__.py | 177 ++++++++++++++++++++++++++++++++++ rwx/sw/yt_dlp/video.py | 0 3 files changed, 291 insertions(+), 79 deletions(-) create mode 100644 rwx/sw/yt_dlp/__init__.py create mode 100644 rwx/sw/yt_dlp/video.py diff --git a/rwx/sw/youtube/__init__.py b/rwx/sw/youtube/__init__.py index 06f5bdc..2db1edb 100644 --- a/rwx/sw/youtube/__init__.py +++ b/rwx/sw/youtube/__init__.py @@ -1,4 +1,4 @@ -import yt_dlp +"""YouTube DownLoad.""" # playlists # … @@ -14,7 +14,6 @@ import yt_dlp # playlist / entries # … - # videos # id # channel @@ -36,8 +35,80 @@ import yt_dlp # thumbnails # view_count +from abc import ABC, abstractmethod +from enum import Enum +from rwx import Object +from rwx.log import stream as log + +import yt_dlp + + +class Action(Enum): + DOWNLOAD_VIDEO = 0 + EXTRACT_PLAYLIST = 1 + EXTRACT_PLAYLISTS = 2 + EXTRACT_VIDEO = 3 + EXTRACT_VIDEOS = 4 + + +class Tab(Object, ABC): + """YouTube Tab.""" + + URL_ROOT = "https://youtube.com" + + def __init__(self, object_id: str) -> None: + """Set object id. + + :param object_id: object identifier + :type object_id: str + """ + self.object_id = object_id + log.info(self.object_id) + self.url = self.get_url() + log.info(self.url) + + def get_options(self) -> dict: + """Return options for the action. + + :rtype: dict + """ + return { + "extract_flat": True, + "skip_download": True, + } + + @abstractmethod + def get_url(self) -> str: + """Return URL to access for object. + + :rtype: str + """ + + @staticmethod + def dl(opt: dict) -> yt_dlp.YoutubeDL: + options = {**opt, "ignoreerrors": False, "quiet": False} + log.info(options) + return yt_dlp.YoutubeDL(options) + + +class Playlist(Tab): + + def __init__(self, playlist_id: str) -> None: + self.playlist_id = playlist_id + + def get_url(self) -> str: + return f"{Tab.URL_ROOT}/playlist?list={self.playlist_id}" + + +class Playlists(Tab): + + def __init__(self, channel_id: str) -> None: + self.channel_id = channel_id + + def get_url(self) -> str: + return f"{Tab.URL_ROOT}/channel/{self.channel_id}/playlists" + -# video # id # title # formats @@ -50,93 +121,57 @@ import yt_dlp # categories # tags # fulltitle +class Video(Tab): + + def __init__(self, video_id: str) -> None: + self.video_id = video_id + + def get_url(self) -> str: + return f"{Tab.URL_ROOT}/watch?v={self.video_id}" -def extract_playlist(playlist_id): - options = { - "quiet": False, - "extract_flat": True, - "force_generic_extractor": False, - "ignoreerrors": False, - } - url = f"https://youtube.com/playlist?list={playlist_id}" - print(f"""\ -{options} +class Videos(Tab): -{url} + def __init__(self, channel_id: str) -> None: + self.channel_id = channel_id -""") - with yt_dlp.YoutubeDL(options) as y: - return y.extract_info(url, download=False) + def get_url(self) -> str: + return f"{Tab.URL_ROOT}/channel/{self.channel_id}/videos" -def extract_playlists(channel_id): - options = { - "quiet": False, - "extract_flat": True, - "force_generic_extractor": False, - "ignoreerrors": False, - "skip_download": True, - } - url = f"https://youtube.com/channel/{channel_id}/playlists" - print(f"""\ -{options} - -{url} - -""") - with yt_dlp.YoutubeDL(options) as y: - return y.extract_info(url, download=False) +def command(action: Action): + match action: + case Action.DOWNLOAD_VIDEO: + options = { + "format": "bestvideo[ext=mp4]+bestaudio[ext=mp4]", + "outtmpl": "%(id)s.%(ext)s", + "writesubtitles": True, + "writethumbnail": True, + } + log.info(options) + with dl(options) as youtube: + match action: + case Action.DOWNLOAD_VIDEO: + youtube.download([url]) + case _: + youtube.extract_info(url, download=False) -def download_video(video_id): - options = { - "format": "bestvideo[ext=mp4]+bestaudio[ext=mp4]", - "outtmpl": "%(id)s.%(ext)s", - "writesubtitles": True, - "writethumbnail": True, - "ignoreerrors": False, - } - url = f"https://youtu.be/{video_id}" - print(f"""\ -{options} - -{url} - -""") - with yt_dlp.YoutubeDL(options) as y: - return y.download([url]) +def download_video(video_id: str): + return command(Action.DOWNLOAD_VIDEO, video_id) -def extract_video(video_id): - options = { - "quiet": False, - "ignoreerrors": False, - } - url = f"https://youtu.be/{video_id}" - print(f"""\ -{options} - -{url} - -""") - with yt_dlp.YoutubeDL(options) as y: - return y.extract_info(url, download=False) +def extract_playlist(playlist_id: str): + return command(Action.EXTRACT_PLAYLIST, playlist_id) -def extract_videos(channel_id): - options = { - "quiet": False, - "extract_flat": True, - "force_generic_extractor": False, - "ignoreerrors": False, - } - url = f"https://youtube.com/channel/{channel_id}/videos" - print(f"""\ -{options} +def extract_playlists(channel_id: str): + return command(Action.EXTRACT_PLAYLISTS, channel_id) -{url} -""") - with yt_dlp.YoutubeDL(options) as y: - return y.extract_info(url, download=False) +def extract_video(video_id: str): + return command(Action.EXTRACT_VIDEO, video_id) + + +def extract_videos(channel_id: str): + return command(Action.EXTRACT_VIDEOS, channel_id) diff --git a/rwx/sw/yt_dlp/__init__.py b/rwx/sw/yt_dlp/__init__.py new file mode 100644 index 0000000..df744eb --- /dev/null +++ b/rwx/sw/yt_dlp/__init__.py @@ -0,0 +1,177 @@ +"""YouTube.""" + +# playlists +# … +# entries + +# playlists / entries +# title +# id + +# playlist +# entries + +# playlist / entries +# … + +# videos +# id +# channel +# channel_id +# title +# channel_follower_count +# description +# tags +# thumbnails +# uploader_id +# uploader +# entries + +# videos / entries +# id +# title +# description truncated +# duration +# thumbnails +# view_count + +from abc import ABC, abstractmethod +from enum import Enum +from rwx import Object +from rwx.log import stream as log + +import yt_dlp + + +class Action(Enum): + DOWNLOAD_VIDEO = 0 + EXTRACT_PLAYLIST = 1 + EXTRACT_PLAYLISTS = 2 + EXTRACT_VIDEO = 3 + EXTRACT_VIDEOS = 4 + + +class Tab(Object, ABC): + """YouTube Tab.""" + + URL_ROOT = "https://youtube.com" + + def __init__(self, object_id: str) -> None: + """Set object id. + + :param object_id: object identifier + :type object_id: str + """ + self.object_id = object_id + log.info(self.object_id) + self.url = self.get_url() + log.info(self.url) + + def get_options(self) -> dict: + """Return options for the action. + + :rtype: dict + """ + return { + "extract_flat": True, + "skip_download": True, + } + + @abstractmethod + def get_url(self) -> str: + """Return URL to access for object. + + :rtype: str + """ + + @staticmethod + def dl(opt: dict) -> yt_dlp.YoutubeDL: + options = {**opt, "ignoreerrors": False, "quiet": False} + log.info(options) + return yt_dlp.YoutubeDL(options) + + +class Playlist(Tab): + + def __init__(self, playlist_id: str) -> None: + self.playlist_id = playlist_id + + def get_url(self) -> str: + return f"{Tab.URL_ROOT}/playlist?list={self.playlist_id}" + + +class Playlists(Tab): + + def __init__(self, channel_id: str) -> None: + self.channel_id = channel_id + + def get_url(self) -> str: + return f"{Tab.URL_ROOT}/channel/{self.channel_id}/playlists" + + +# id +# title +# formats +# thumbnails +# thumbnail +# description +# +# duration +# view_count +# categories +# tags +# fulltitle +class Video(Tab): + + def __init__(self, video_id: str) -> None: + self.video_id = video_id + + def get_url(self) -> str: + return f"{Tab.URL_ROOT}/watch?v={self.video_id}" + + +class Videos(Tab): + + def __init__(self, channel_id: str) -> None: + self.channel_id = channel_id + + def get_url(self) -> str: + return f"{Tab.URL_ROOT}/channel/{self.channel_id}/videos" + + +def command(action: Action): + match action: + case Action.DOWNLOAD_VIDEO: + options = { + "format": "bestvideo[ext=mp4]+bestaudio[ext=mp4]", + "outtmpl": "%(id)s.%(ext)s", + "writesubtitles": True, + "writethumbnail": True, + } + log.info(options) + with dl(options) as youtube: + match action: + case Action.DOWNLOAD_VIDEO: + youtube.download([url]) + case _: + youtube.extract_info(url, download=False) + + +def download_video(video_id: str): + return command(Action.DOWNLOAD_VIDEO, video_id) + + +def extract_playlist(playlist_id: str): + return command(Action.EXTRACT_PLAYLIST, playlist_id) + + +def extract_playlists(channel_id: str): + return command(Action.EXTRACT_PLAYLISTS, channel_id) + + +def extract_video(video_id: str): + return command(Action.EXTRACT_VIDEO, video_id) + + +def extract_videos(channel_id: str): + return command(Action.EXTRACT_VIDEOS, channel_id) diff --git a/rwx/sw/yt_dlp/video.py b/rwx/sw/yt_dlp/video.py new file mode 100644 index 0000000..e69de29 From 61ea62a2725cbb12d793739fa885f4f1bf72fa42 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 16 Mar 2025 16:17:02 +0100 Subject: [PATCH 05/57] wip --- rwx/sw/youtube/__init__.py | 177 ------------------------------------- rwx/sw/yt_dlp/__init__.py | 54 ++++------- 2 files changed, 15 insertions(+), 216 deletions(-) delete mode 100644 rwx/sw/youtube/__init__.py diff --git a/rwx/sw/youtube/__init__.py b/rwx/sw/youtube/__init__.py deleted file mode 100644 index 2db1edb..0000000 --- a/rwx/sw/youtube/__init__.py +++ /dev/null @@ -1,177 +0,0 @@ -"""YouTube DownLoad.""" - -# playlists -# … -# entries - -# playlists / entries -# title -# id - -# playlist -# entries - -# playlist / entries -# … - -# videos -# id -# channel -# channel_id -# title -# channel_follower_count -# description -# tags -# thumbnails -# uploader_id -# uploader -# entries - -# videos / entries -# id -# title -# description truncated -# duration -# thumbnails -# view_count - -from abc import ABC, abstractmethod -from enum import Enum -from rwx import Object -from rwx.log import stream as log - -import yt_dlp - - -class Action(Enum): - DOWNLOAD_VIDEO = 0 - EXTRACT_PLAYLIST = 1 - EXTRACT_PLAYLISTS = 2 - EXTRACT_VIDEO = 3 - EXTRACT_VIDEOS = 4 - - -class Tab(Object, ABC): - """YouTube Tab.""" - - URL_ROOT = "https://youtube.com" - - def __init__(self, object_id: str) -> None: - """Set object id. - - :param object_id: object identifier - :type object_id: str - """ - self.object_id = object_id - log.info(self.object_id) - self.url = self.get_url() - log.info(self.url) - - def get_options(self) -> dict: - """Return options for the action. - - :rtype: dict - """ - return { - "extract_flat": True, - "skip_download": True, - } - - @abstractmethod - def get_url(self) -> str: - """Return URL to access for object. - - :rtype: str - """ - - @staticmethod - def dl(opt: dict) -> yt_dlp.YoutubeDL: - options = {**opt, "ignoreerrors": False, "quiet": False} - log.info(options) - return yt_dlp.YoutubeDL(options) - - -class Playlist(Tab): - - def __init__(self, playlist_id: str) -> None: - self.playlist_id = playlist_id - - def get_url(self) -> str: - return f"{Tab.URL_ROOT}/playlist?list={self.playlist_id}" - - -class Playlists(Tab): - - def __init__(self, channel_id: str) -> None: - self.channel_id = channel_id - - def get_url(self) -> str: - return f"{Tab.URL_ROOT}/channel/{self.channel_id}/playlists" - - -# id -# title -# formats -# thumbnails -# thumbnail -# description -# -# duration -# view_count -# categories -# tags -# fulltitle -class Video(Tab): - - def __init__(self, video_id: str) -> None: - self.video_id = video_id - - def get_url(self) -> str: - return f"{Tab.URL_ROOT}/watch?v={self.video_id}" - - -class Videos(Tab): - - def __init__(self, channel_id: str) -> None: - self.channel_id = channel_id - - def get_url(self) -> str: - return f"{Tab.URL_ROOT}/channel/{self.channel_id}/videos" - - -def command(action: Action): - match action: - case Action.DOWNLOAD_VIDEO: - options = { - "format": "bestvideo[ext=mp4]+bestaudio[ext=mp4]", - "outtmpl": "%(id)s.%(ext)s", - "writesubtitles": True, - "writethumbnail": True, - } - log.info(options) - with dl(options) as youtube: - match action: - case Action.DOWNLOAD_VIDEO: - youtube.download([url]) - case _: - youtube.extract_info(url, download=False) - - -def download_video(video_id: str): - return command(Action.DOWNLOAD_VIDEO, video_id) - - -def extract_playlist(playlist_id: str): - return command(Action.EXTRACT_PLAYLIST, playlist_id) - - -def extract_playlists(channel_id: str): - return command(Action.EXTRACT_PLAYLISTS, channel_id) - - -def extract_video(video_id: str): - return command(Action.EXTRACT_VIDEO, video_id) - - -def extract_videos(channel_id: str): - return command(Action.EXTRACT_VIDEOS, channel_id) diff --git a/rwx/sw/yt_dlp/__init__.py b/rwx/sw/yt_dlp/__init__.py index df744eb..3e44298 100644 --- a/rwx/sw/yt_dlp/__init__.py +++ b/rwx/sw/yt_dlp/__init__.py @@ -1,4 +1,4 @@ -"""YouTube.""" +"""YouTube DownLoad.""" # playlists # … @@ -39,8 +39,7 @@ from abc import ABC, abstractmethod from enum import Enum from rwx import Object from rwx.log import stream as log - -import yt_dlp +from yt_dlp import YoutubeDL class Action(Enum): @@ -66,6 +65,7 @@ class Tab(Object, ABC): log.info(self.object_id) self.url = self.get_url() log.info(self.url) + self.info = Tab.yt_dl(self.get_options()).extract_info(self.url, download=False) def get_options(self) -> dict: """Return options for the action. @@ -85,14 +85,13 @@ class Tab(Object, ABC): """ @staticmethod - def dl(opt: dict) -> yt_dlp.YoutubeDL: + def yt_dl(opt: dict) -> YoutubeDL: options = {**opt, "ignoreerrors": False, "quiet": False} log.info(options) - return yt_dlp.YoutubeDL(options) + return YoutubeDL(options) class Playlist(Tab): - def __init__(self, playlist_id: str) -> None: self.playlist_id = playlist_id @@ -101,7 +100,6 @@ class Playlist(Tab): class Playlists(Tab): - def __init__(self, channel_id: str) -> None: self.channel_id = channel_id @@ -122,16 +120,24 @@ class Playlists(Tab): # tags # fulltitle class Video(Tab): - def __init__(self, video_id: str) -> None: self.video_id = video_id + def download(self) -> None: + Tab.yt_dl( + { + "format": "bestvideo[ext=mp4]+bestaudio[ext=mp4]", + "outtmpl": "%(id)s.%(ext)s", + "writesubtitles": True, + "writethumbnail": True, + } + ).download([self.url]) + def get_url(self) -> str: return f"{Tab.URL_ROOT}/watch?v={self.video_id}" class Videos(Tab): - def __init__(self, channel_id: str) -> None: self.channel_id = channel_id @@ -140,38 +146,8 @@ class Videos(Tab): def command(action: Action): - match action: - case Action.DOWNLOAD_VIDEO: - options = { - "format": "bestvideo[ext=mp4]+bestaudio[ext=mp4]", - "outtmpl": "%(id)s.%(ext)s", - "writesubtitles": True, - "writethumbnail": True, - } log.info(options) with dl(options) as youtube: match action: - case Action.DOWNLOAD_VIDEO: - youtube.download([url]) case _: youtube.extract_info(url, download=False) - - -def download_video(video_id: str): - return command(Action.DOWNLOAD_VIDEO, video_id) - - -def extract_playlist(playlist_id: str): - return command(Action.EXTRACT_PLAYLIST, playlist_id) - - -def extract_playlists(channel_id: str): - return command(Action.EXTRACT_PLAYLISTS, channel_id) - - -def extract_video(video_id: str): - return command(Action.EXTRACT_VIDEO, video_id) - - -def extract_videos(channel_id: str): - return command(Action.EXTRACT_VIDEOS, channel_id) From 6a2421922b8bd91c4a57b30d2e31d0035fb10828 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 16 Mar 2025 17:40:45 +0100 Subject: [PATCH 06/57] wip --- rwx/sw/yt_dlp/__init__.py | 37 +++++++++++-------------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/rwx/sw/yt_dlp/__init__.py b/rwx/sw/yt_dlp/__init__.py index 3e44298..b949bfc 100644 --- a/rwx/sw/yt_dlp/__init__.py +++ b/rwx/sw/yt_dlp/__init__.py @@ -36,20 +36,11 @@ # view_count from abc import ABC, abstractmethod -from enum import Enum from rwx import Object from rwx.log import stream as log from yt_dlp import YoutubeDL -class Action(Enum): - DOWNLOAD_VIDEO = 0 - EXTRACT_PLAYLIST = 1 - EXTRACT_PLAYLISTS = 2 - EXTRACT_VIDEO = 3 - EXTRACT_VIDEOS = 4 - - class Tab(Object, ABC): """YouTube Tab.""" @@ -65,7 +56,9 @@ class Tab(Object, ABC): log.info(self.object_id) self.url = self.get_url() log.info(self.url) - self.info = Tab.yt_dl(self.get_options()).extract_info(self.url, download=False) + self.info = Tab.yt_dl(self.get_options()).extract_info( + self.url, download=False + ) def get_options(self) -> dict: """Return options for the action. @@ -93,18 +86,18 @@ class Tab(Object, ABC): class Playlist(Tab): def __init__(self, playlist_id: str) -> None: - self.playlist_id = playlist_id + super().__init__(playlist_id) def get_url(self) -> str: - return f"{Tab.URL_ROOT}/playlist?list={self.playlist_id}" + return f"{Tab.URL_ROOT}/playlist?list={self.object_id}" class Playlists(Tab): def __init__(self, channel_id: str) -> None: - self.channel_id = channel_id + super().__init__(channel_id) def get_url(self) -> str: - return f"{Tab.URL_ROOT}/channel/{self.channel_id}/playlists" + return f"{Tab.URL_ROOT}/channel/{self.object_id}/playlists" # id @@ -121,7 +114,7 @@ class Playlists(Tab): # fulltitle class Video(Tab): def __init__(self, video_id: str) -> None: - self.video_id = video_id + super().__init__(video_id) def download(self) -> None: Tab.yt_dl( @@ -134,20 +127,12 @@ class Video(Tab): ).download([self.url]) def get_url(self) -> str: - return f"{Tab.URL_ROOT}/watch?v={self.video_id}" + return f"{Tab.URL_ROOT}/watch?v={self.object_id}" class Videos(Tab): def __init__(self, channel_id: str) -> None: - self.channel_id = channel_id + super().__init__(channel_id) def get_url(self) -> str: - return f"{Tab.URL_ROOT}/channel/{self.channel_id}/videos" - - -def command(action: Action): - log.info(options) - with dl(options) as youtube: - match action: - case _: - youtube.extract_info(url, download=False) + return f"{Tab.URL_ROOT}/channel/{self.object_id}/videos" From 9e37e3be95e35c3c1ce545d1b535522d815c95e7 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 16 Mar 2025 18:28:40 +0100 Subject: [PATCH 07/57] extract --- rwx/sw/yt_dlp/__init__.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/rwx/sw/yt_dlp/__init__.py b/rwx/sw/yt_dlp/__init__.py index b949bfc..f8db074 100644 --- a/rwx/sw/yt_dlp/__init__.py +++ b/rwx/sw/yt_dlp/__init__.py @@ -56,7 +56,14 @@ class Tab(Object, ABC): log.info(self.object_id) self.url = self.get_url() log.info(self.url) - self.info = Tab.yt_dl(self.get_options()).extract_info( + + def extract(self) -> dict: + """Return extracted dict. + + :rtype: dict + """ + yt_dl = Tab.yt_dl(self.get_options()) + return yt_dl.extract_info( self.url, download=False ) @@ -100,21 +107,19 @@ class Playlists(Tab): return f"{Tab.URL_ROOT}/channel/{self.object_id}/playlists" -# id -# title -# formats -# thumbnails -# thumbnail -# description -# -# duration -# view_count -# categories -# tags -# fulltitle class Video(Tab): def __init__(self, video_id: str) -> None: super().__init__(video_id) + info = self.extract() + self.title = info["title"] + self.fulltitle = info["fulltitle"] + self.duration = info["duration"] + self.categories = info["categories"] + self.tags = info["tags"] + self.description = info["description"] + # TODO formats + # TODO thumbnails + # TODO thumbnail def download(self) -> None: Tab.yt_dl( From 1f046e4ba94b9e1aeb653d580418192d0112e294 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 16 Mar 2025 22:30:37 +0100 Subject: [PATCH 08/57] any --- rwx/sw/yt_dlp/__init__.py | 53 +++++++++++++++++++++------------------ rwx/web/__init__.py | 24 ++++++++++++++++++ 2 files changed, 52 insertions(+), 25 deletions(-) create mode 100644 rwx/web/__init__.py diff --git a/rwx/sw/yt_dlp/__init__.py b/rwx/sw/yt_dlp/__init__.py index f8db074..9e261ff 100644 --- a/rwx/sw/yt_dlp/__init__.py +++ b/rwx/sw/yt_dlp/__init__.py @@ -14,28 +14,8 @@ # playlist / entries # … -# videos -# id -# channel -# channel_id -# title -# channel_follower_count -# description -# tags -# thumbnails -# uploader_id -# uploader -# entries - -# videos / entries -# id -# title -# description truncated -# duration -# thumbnails -# view_count - from abc import ABC, abstractmethod +from typing import Any from rwx import Object from rwx.log import stream as log from yt_dlp import YoutubeDL @@ -57,15 +37,13 @@ class Tab(Object, ABC): self.url = self.get_url() log.info(self.url) - def extract(self) -> dict: + def extract(self) -> dict[str, Any]: """Return extracted dict. :rtype: dict """ yt_dl = Tab.yt_dl(self.get_options()) - return yt_dl.extract_info( - self.url, download=False - ) + return yt_dl.extract_info(self.url, download=False) def get_options(self) -> dict: """Return options for the action. @@ -135,9 +113,34 @@ class Video(Tab): return f"{Tab.URL_ROOT}/watch?v={self.object_id}" +# channel +# title +# channel_follower_count +# description +# tags +# thumbnails +# uploader_id +# uploader +# videos / entries +# title +# description truncated +# duration +# thumbnails +# view_count class Videos(Tab): def __init__(self, channel_id: str) -> None: super().__init__(channel_id) + info = self.extract() + self.title = info["title"] + self.ids = [v["id"] for v in info["entries"]] + self.videos = {} def get_url(self) -> str: return f"{Tab.URL_ROOT}/channel/{self.object_id}/videos" + + def load(self) -> None: + done = 0 + for video_id in self.ids: + self.videos[video_id] = Video(video_id) + done += 1 + log.info(done) diff --git a/rwx/web/__init__.py b/rwx/web/__init__.py new file mode 100644 index 0000000..943a3f4 --- /dev/null +++ b/rwx/web/__init__.py @@ -0,0 +1,24 @@ +from rwx import Object, txt +from rwx.txt import CHARSET + + +class Page(Object): + def __init__(self): + self.charset = CHARSET + self.description = "" + self.title = "" + + def render(self) -> str: + return f"""\ + + + + + + +{self.title} + + + + +""" From 5b091397d25f66e249f4377f7ff31ed210e134b4 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Mon, 17 Mar 2025 14:53:47 +0100 Subject: [PATCH 09/57] download,next --- rwx/sw/yt_dlp/__init__.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/rwx/sw/yt_dlp/__init__.py b/rwx/sw/yt_dlp/__init__.py index 9e261ff..d28ab86 100644 --- a/rwx/sw/yt_dlp/__init__.py +++ b/rwx/sw/yt_dlp/__init__.py @@ -15,6 +15,7 @@ # … from abc import ABC, abstractmethod +from pathlib import Path from typing import Any from rwx import Object from rwx.log import stream as log @@ -64,7 +65,11 @@ class Tab(Object, ABC): @staticmethod def yt_dl(opt: dict) -> YoutubeDL: - options = {**opt, "ignoreerrors": False, "quiet": False} + options = { + **opt, + "ignoreerrors": False, + "quiet": False, + } log.info(options) return YoutubeDL(options) @@ -95,9 +100,6 @@ class Video(Tab): self.categories = info["categories"] self.tags = info["tags"] self.description = info["description"] - # TODO formats - # TODO thumbnails - # TODO thumbnail def download(self) -> None: Tab.yt_dl( @@ -132,7 +134,7 @@ class Videos(Tab): super().__init__(channel_id) info = self.extract() self.title = info["title"] - self.ids = [v["id"] for v in info["entries"]] + self.ids = [v["id"] for v in reversed(info["entries"])] self.videos = {} def get_url(self) -> str: @@ -144,3 +146,21 @@ class Videos(Tab): self.videos[video_id] = Video(video_id) done += 1 log.info(done) + + def next(self) -> str | None: + for video_id in self.ids: + if not Path(f"{video_id}.mp4").exists(): + return video_id + return None + + +def download(video_id: str | None) -> None: + if video_id: + Tab.yt_dl( + { + "format": "bestvideo[ext=mp4]+bestaudio[ext=mp4]", + "outtmpl": "%(id)s.%(ext)s", + "writesubtitles": True, + "writethumbnail": True, + } + ).download([f"{Tab.URL_ROOT}/watch?v={video_id}"]) From 46073e6e3a20d91bfb11fd9eedf2fc157ff0078d Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Mon, 17 Mar 2025 19:49:35 +0100 Subject: [PATCH 10/57] sponsors --- rwx/sw/yt_dlp/__init__.py | 83 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/rwx/sw/yt_dlp/__init__.py b/rwx/sw/yt_dlp/__init__.py index d28ab86..e3c2bbd 100644 --- a/rwx/sw/yt_dlp/__init__.py +++ b/rwx/sw/yt_dlp/__init__.py @@ -21,6 +21,8 @@ from rwx import Object from rwx.log import stream as log from yt_dlp import YoutubeDL +URL = "https://youtube.com" + class Tab(Object, ABC): """YouTube Tab.""" @@ -154,13 +156,88 @@ class Videos(Tab): return None -def download(video_id: str | None) -> None: +def download_video(video_id: str | None) -> None: if video_id: - Tab.yt_dl( + ytdl( { "format": "bestvideo[ext=mp4]+bestaudio[ext=mp4]", "outtmpl": "%(id)s.%(ext)s", + "postprocessors": [ + { + "key": "SponsorBlock", + "categories": ["sponsor"], + }, + { + "key": "ModifyChapters", + "remove_sponsor_segments": ["sponsor"], + }, + ], "writesubtitles": True, "writethumbnail": True, } - ).download([f"{Tab.URL_ROOT}/watch?v={video_id}"]) + ).download([f"{URL}/watch?v={video_id}"]) + + +def extract(opt: dict, url: str) -> dict: + """Return extracted dict. + + :rtype: dict + """ + return ytdl({ + **opt, + "extract_flat": True, + "skip_download": True, + }).extract_info(url, download=False) + + +def extract_channel(channel_id: str) -> dict: + """Return extracted channel dict. + + :rtype: dict + """ + d = extract({ + }, f"{URL}/channel/{channel_id}") + log.info(d) + return { + } + + +def extract_channel_videos(channel_id: str) -> list[str]: + """Return extracted channel videos dict. + + :rtype: dict + """ + d = extract({ + }, f"{URL}/channel/{channel_id}/videos") + log.info(d) + return [entry["id"] for entry in reversed(d["entries"])] + + +def extract_video(video_id: str) -> dict: + """Return extracted video dict. + + :rtype: dict + """ + d = extract({ + }, f"{URL}/watch?v={video_id}") + log.info(d) + return { + "title": d["title"], + } + + +def next_download(videos: list[str]) -> str | None: + for video_id in videos: + if not Path(f"{video_id}.mp4").exists(): + return video_id + return None + + +def ytdl(opt: dict) -> YoutubeDL: + options = { + **opt, + "ignoreerrors": False, + "quiet": False, + } + log.info(options) + return YoutubeDL(options) From 1c44e0cc7f9e85cac532f57aa10e461cec1eca13 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Mon, 17 Mar 2025 20:30:12 +0100 Subject: [PATCH 11/57] next --- rwx/sw/yt_dlp/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rwx/sw/yt_dlp/__init__.py b/rwx/sw/yt_dlp/__init__.py index e3c2bbd..d884724 100644 --- a/rwx/sw/yt_dlp/__init__.py +++ b/rwx/sw/yt_dlp/__init__.py @@ -227,8 +227,11 @@ def extract_video(video_id: str) -> dict: def next_download(videos: list[str]) -> str | None: + index = 0 for video_id in videos: + index += 1 if not Path(f"{video_id}.mp4").exists(): + log.info(f"{index} ∕ {len(videos)}") return video_id return None From 3f41a52e56cb5f48d14558887056104eeca226dd Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 09:04:47 +0100 Subject: [PATCH 12/57] filters --- rwx/sw/yt_dlp/__init__.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/rwx/sw/yt_dlp/__init__.py b/rwx/sw/yt_dlp/__init__.py index d884724..7308868 100644 --- a/rwx/sw/yt_dlp/__init__.py +++ b/rwx/sw/yt_dlp/__init__.py @@ -202,14 +202,14 @@ def extract_channel(channel_id: str) -> dict: } -def extract_channel_videos(channel_id: str) -> list[str]: +def extract_videos(channel_id: str) -> list[str]: """Return extracted channel videos dict. :rtype: dict """ d = extract({ }, f"{URL}/channel/{channel_id}/videos") - log.info(d) + return d return [entry["id"] for entry in reversed(d["entries"])] @@ -220,12 +220,31 @@ def extract_video(video_id: str) -> dict: """ d = extract({ }, f"{URL}/watch?v={video_id}") - log.info(d) + return d + + +def filter_video(d: dict) -> dict: + """Return filtered video dict. + + :param d: video info dict + :type d: dict + :rtype: dict + """ return { "title": d["title"], } +def filter_videos(d: dict) -> list[str]: + """Return filtered videos dict. + + :param d: videos dict + :type d: dict + :rtype: dict + """ + return [entry["id"] for entry in reversed(d["entries"])] + + def next_download(videos: list[str]) -> str | None: index = 0 for video_id in videos: From e75692ee26dcf32c96b51d6c35f8e9a638eaba78 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 09:05:30 +0100 Subject: [PATCH 13/57] ytdlp --- rwx/sw/{yt_dlp => ytdlp}/__init__.py | 0 rwx/sw/{yt_dlp => ytdlp}/video.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename rwx/sw/{yt_dlp => ytdlp}/__init__.py (100%) rename rwx/sw/{yt_dlp => ytdlp}/video.py (100%) diff --git a/rwx/sw/yt_dlp/__init__.py b/rwx/sw/ytdlp/__init__.py similarity index 100% rename from rwx/sw/yt_dlp/__init__.py rename to rwx/sw/ytdlp/__init__.py diff --git a/rwx/sw/yt_dlp/video.py b/rwx/sw/ytdlp/video.py similarity index 100% rename from rwx/sw/yt_dlp/video.py rename to rwx/sw/ytdlp/video.py From 899b1383de71345c87d8ee4f8c89771a5e5ef191 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 11:41:06 +0100 Subject: [PATCH 14/57] lint --- rwx/sw/ytdlp/__init__.py | 131 ++++++++++++++++++++++++++++++--------- 1 file changed, 103 insertions(+), 28 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 7308868..28bda2b 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -17,13 +17,28 @@ from abc import ABC, abstractmethod from pathlib import Path from typing import Any -from rwx import Object -from rwx.log import stream as log from yt_dlp import YoutubeDL +from rwx import Object +from rwx.log import stream as log + URL = "https://youtube.com" +class Channel(Object): + def __init__(self, d: dict) -> None: + self.identifier = d["channel_id"] + self.title = d["channel"] + self.followers = int(d["channel_follower_count"]) + self.description = d["description"] + self.tags = d["tags"] + # TODO thumbnails + self.uploader_id = d["uploader_id"] + self.uploader = d["uploader"] + self.url = d["channel_url"] + # TODO entries + + class Tab(Object, ABC): """YouTube Tab.""" @@ -156,6 +171,11 @@ class Videos(Tab): return None +# ╭──────────╮ +# │ download │ +# ╰──────────╯ + + def download_video(video_id: str | None) -> None: if video_id: ytdl( @@ -178,51 +198,63 @@ def download_video(video_id: str | None) -> None: ).download([f"{URL}/watch?v={video_id}"]) -def extract(opt: dict, url: str) -> dict: +# ╭─────────╮ +# │ extract │ +# ╰─────────╯ + + +def extract(opt: dict[str, Any], url: str) -> dict[str, Any]: """Return extracted dict. :rtype: dict """ - return ytdl({ - **opt, - "extract_flat": True, - "skip_download": True, - }).extract_info(url, download=False) + return ytdl( + { + **opt, + "extract_flat": True, + "skip_download": True, + } + ).extract_info(url, download=False) def extract_channel(channel_id: str) -> dict: """Return extracted channel dict. + :param channel_id: channel identifier + :type channel_id: str :rtype: dict """ - d = extract({ - }, f"{URL}/channel/{channel_id}") - log.info(d) - return { - } - - -def extract_videos(channel_id: str) -> list[str]: - """Return extracted channel videos dict. - - :rtype: dict - """ - d = extract({ - }, f"{URL}/channel/{channel_id}/videos") + d = extract({}, f"{URL}/channel/{channel_id}") + return d + + +def extract_videos(channel_id: str) -> dict: + """Return extracted videos dict. + + :param channel_id: channel identifier + :type channel_id: str + :rtype: dict + """ + d = extract({}, f"{URL}/channel/{channel_id}/videos") return d - return [entry["id"] for entry in reversed(d["entries"])] def extract_video(video_id: str) -> dict: """Return extracted video dict. + :param video_id: video identifier + :type video_id: str :rtype: dict """ - d = extract({ - }, f"{URL}/watch?v={video_id}") + d = extract({}, f"{URL}/watch?v={video_id}") return d +# ╭────────╮ +# │ filter │ +# ╰────────╯ + + def filter_video(d: dict) -> dict: """Return filtered video dict. @@ -245,16 +277,59 @@ def filter_videos(d: dict) -> list[str]: return [entry["id"] for entry in reversed(d["entries"])] +# ╭──────╮ +# │ next │ +# ╰──────╯ + + def next_download(videos: list[str]) -> str | None: - index = 0 - for video_id in videos: - index += 1 + for index, video_id in enumerate(videos): if not Path(f"{video_id}.mp4").exists(): log.info(f"{index} ∕ {len(videos)}") return video_id return None +# ╭─────╮ +# │ url │ +# ╰─────╯ + + +def url_channel(channel_id: str) -> str: + """Return channel URL. + + :param channel_id: channel identifier + :type channel_id: str + :rtype: str + """ + return f"{URL}/channel/{channel_id}" + + +def url_playlists(channel_id: str) -> str: + """Return playlists URL. + + :param channel_id: channel identifier + :type channel_id: str + :rtype: str + """ + return f"{url_channel(channel_id)}/playlists" + + +def url_videos(channel_id: str) -> str: + """Return videos URL. + + :param channel_id: channel identifier + :type channel_id: str + :rtype: str + """ + return f"{url_channel(channel_id)}/videos" + + +# ╭──────╮ +# │ ytdl │ +# ╰──────╯ + + def ytdl(opt: dict) -> YoutubeDL: options = { **opt, From 75d54d3dbf561be76c61c79dbee8c0d950516ad7 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 11:45:31 +0100 Subject: [PATCH 15/57] lint --- rwx/fs/__init__.py | 3 +-- rwx/sw/ytdlp/__init__.py | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rwx/fs/__init__.py b/rwx/fs/__init__.py index 8a45288..1873da2 100644 --- a/rwx/fs/__init__.py +++ b/rwx/fs/__init__.py @@ -2,9 +2,8 @@ import os import shutil -from pathlib import Path - import tomllib +from pathlib import Path from rwx import ps diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 28bda2b..8d6ce3f 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -17,6 +17,7 @@ from abc import ABC, abstractmethod from pathlib import Path from typing import Any + from yt_dlp import YoutubeDL from rwx import Object @@ -125,7 +126,7 @@ class Video(Tab): "outtmpl": "%(id)s.%(ext)s", "writesubtitles": True, "writethumbnail": True, - } + }, ).download([self.url]) def get_url(self) -> str: @@ -194,7 +195,7 @@ def download_video(video_id: str | None) -> None: ], "writesubtitles": True, "writethumbnail": True, - } + }, ).download([f"{URL}/watch?v={video_id}"]) @@ -213,7 +214,7 @@ def extract(opt: dict[str, Any], url: str) -> dict[str, Any]: **opt, "extract_flat": True, "skip_download": True, - } + }, ).extract_info(url, download=False) From e81cd5b745872f8c90adf39f5c58223479a7ebd4 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 16:06:46 +0100 Subject: [PATCH 16/57] wip/cache --- rwx/sw/ytdlp/__init__.py | 259 +++++++++++++++++---------------------- 1 file changed, 110 insertions(+), 149 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 8d6ce3f..c6bae2b 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -1,20 +1,6 @@ """YouTube DownLoad.""" -# playlists -# … -# entries - -# playlists / entries -# title -# id - -# playlist -# entries - -# playlist / entries -# … - -from abc import ABC, abstractmethod +from datetime import datetime from pathlib import Path from typing import Any @@ -26,9 +12,25 @@ from rwx.log import stream as log URL = "https://youtube.com" +class Cache(Object): + """YouTube local cache.""" + + def __init__(self, root: Path) -> None: + self.root = root + + class Channel(Object): - def __init__(self, d: dict) -> None: - self.identifier = d["channel_id"] + """YouTube channel.""" + + def __init__(self, channel_id: str) -> None: + """Set objects tree. + + :param channel_id: channel identifier + :type channel_id: str + """ + d = extract_videos(channel_id) + # channel + self.uid = d["channel_id"] self.title = d["channel"] self.followers = int(d["channel_follower_count"]) self.description = d["description"] @@ -36,101 +38,54 @@ class Channel(Object): # TODO thumbnails self.uploader_id = d["uploader_id"] self.uploader = d["uploader"] - self.url = d["channel_url"] - # TODO entries + # videos + self.videos_ids = [video["id"] for video in reversed(d["entries"])] + # TODO filter members-only + # playlists + d = extract_playlists(channel_id) + self.playlists_ids = [playlist["id"] for playlist in reversed(d["entries"])] + def add_video(self, d: dict) -> dict: + """Add video extra info.""" + self.video_index += 1 + log.info(f"{self.video_index} ∕ {len(self.videos_ids)}") + self.videos.append(Video(d)) + return d -class Tab(Object, ABC): - """YouTube Tab.""" - - URL_ROOT = "https://youtube.com" - - def __init__(self, object_id: str) -> None: - """Set object id. - - :param object_id: object identifier - :type object_id: str - """ - self.object_id = object_id - log.info(self.object_id) - self.url = self.get_url() - log.info(self.url) - - def extract(self) -> dict[str, Any]: - """Return extracted dict. - - :rtype: dict - """ - yt_dl = Tab.yt_dl(self.get_options()) - return yt_dl.extract_info(self.url, download=False) - - def get_options(self) -> dict: - """Return options for the action. - - :rtype: dict - """ - return { - "extract_flat": True, - "skip_download": True, - } - - @abstractmethod - def get_url(self) -> str: - """Return URL to access for object. - - :rtype: str - """ - - @staticmethod - def yt_dl(opt: dict) -> YoutubeDL: - options = { - **opt, - "ignoreerrors": False, - "quiet": False, - } - log.info(options) - return YoutubeDL(options) - - -class Playlist(Tab): - def __init__(self, playlist_id: str) -> None: - super().__init__(playlist_id) - - def get_url(self) -> str: - return f"{Tab.URL_ROOT}/playlist?list={self.object_id}" - - -class Playlists(Tab): - def __init__(self, channel_id: str) -> None: - super().__init__(channel_id) - - def get_url(self) -> str: - return f"{Tab.URL_ROOT}/channel/{self.object_id}/playlists" - - -class Video(Tab): - def __init__(self, video_id: str) -> None: - super().__init__(video_id) - info = self.extract() - self.title = info["title"] - self.fulltitle = info["fulltitle"] - self.duration = info["duration"] - self.categories = info["categories"] - self.tags = info["tags"] - self.description = info["description"] - - def download(self) -> None: - Tab.yt_dl( + def load_videos(self) -> None: + """Load videos extra info.""" + self.videos = [] + # a + #for index, video_id in enumerate(self.videos_ids): + # log.info(f"{index} ∕ {len(self.videos_ids)}") + # self.videos.append(Video(video_id)) + # b + videos_urls = [url_video(video_id) for video_id in self.videos_ids] + y = ytdl( { - "format": "bestvideo[ext=mp4]+bestaudio[ext=mp4]", - "outtmpl": "%(id)s.%(ext)s", - "writesubtitles": True, - "writethumbnail": True, + "process_info_hooks": [self.add_video], + "skip_download": True, }, - ).download([self.url]) + ) + y.download(videos_urls) - def get_url(self) -> str: - return f"{Tab.URL_ROOT}/watch?v={self.object_id}" + +class Video(Object): + """YouTube video.""" + + def __init__(self, d: dict) -> None: + """Set video info. + + :param d: video info + :type d: dict + """ + self.datetime = datetime.now().strftime("%Y%m%d%H%M%S") + self.uid = d["id"] + self.fulltitle = d["fulltitle"] + self.duration = d["duration"] + self.categories = d["categories"] + self.tags = d["tags"] + self.description = d["description"] # channel @@ -147,29 +102,6 @@ class Video(Tab): # duration # thumbnails # view_count -class Videos(Tab): - def __init__(self, channel_id: str) -> None: - super().__init__(channel_id) - info = self.extract() - self.title = info["title"] - self.ids = [v["id"] for v in reversed(info["entries"])] - self.videos = {} - - def get_url(self) -> str: - return f"{Tab.URL_ROOT}/channel/{self.object_id}/videos" - - def load(self) -> None: - done = 0 - for video_id in self.ids: - self.videos[video_id] = Video(video_id) - done += 1 - log.info(done) - - def next(self) -> str | None: - for video_id in self.ids: - if not Path(f"{video_id}.mp4").exists(): - return video_id - return None # ╭──────────╮ @@ -196,7 +128,7 @@ def download_video(video_id: str | None) -> None: "writesubtitles": True, "writethumbnail": True, }, - ).download([f"{URL}/watch?v={video_id}"]) + ).download([url_video(video_id)]) # ╭─────────╮ @@ -209,35 +141,35 @@ def extract(opt: dict[str, Any], url: str) -> dict[str, Any]: :rtype: dict """ - return ytdl( + d = ytdl( { **opt, "extract_flat": True, "skip_download": True, }, ).extract_info(url, download=False) + log.info(d) + return d -def extract_channel(channel_id: str) -> dict: - """Return extracted channel dict. +def extract_playlist(playlist_id: str) -> dict: + """Return extracted playlist dict. + + :param playlist_id: playlist identifier + :type playlist_id: str + :rtype: dict + """ + return extract({}, url_playlist(playlist_id)) + + +def extract_playlists(channel_id: str) -> dict: + """Return extracted playlists dict. :param channel_id: channel identifier :type channel_id: str :rtype: dict """ - d = extract({}, f"{URL}/channel/{channel_id}") - return d - - -def extract_videos(channel_id: str) -> dict: - """Return extracted videos dict. - - :param channel_id: channel identifier - :type channel_id: str - :rtype: dict - """ - d = extract({}, f"{URL}/channel/{channel_id}/videos") - return d + return extract({}, url_playlists(channel_id)) def extract_video(video_id: str) -> dict: @@ -247,8 +179,17 @@ def extract_video(video_id: str) -> dict: :type video_id: str :rtype: dict """ - d = extract({}, f"{URL}/watch?v={video_id}") - return d + return extract({}, url_video(video_id)) + + +def extract_videos(channel_id: str) -> dict: + """Return extracted videos dict. + + :param channel_id: channel identifier + :type channel_id: str + :rtype: dict + """ + return extract({}, url_videos(channel_id)) # ╭────────╮ @@ -306,6 +247,16 @@ def url_channel(channel_id: str) -> str: return f"{URL}/channel/{channel_id}" +def url_playlist(playlist_id: str) -> str: + """Return playlist URL. + + :param playlist_id: playlist identifier + :type playlist_id: str + :rtype: str + """ + return f"{URL}/playlist?list={playlist_id}" + + def url_playlists(channel_id: str) -> str: """Return playlists URL. @@ -316,6 +267,16 @@ def url_playlists(channel_id: str) -> str: return f"{url_channel(channel_id)}/playlists" +def url_video(video_id: str) -> str: + """Return video URL. + + :param video_id: video identifier + :type video_id: str + :rtype: str + """ + return f"{URL}/watch?v={video_id}" + + def url_videos(channel_id: str) -> str: """Return videos URL. From cc4eb0e686062079d850b5b35597350f9870cd94 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 16:30:21 +0100 Subject: [PATCH 17/57] useless --- rwx/sw/ytdlp/__init__.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index c6bae2b..326d5b1 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -12,6 +12,11 @@ from rwx.log import stream as log URL = "https://youtube.com" +# ╭─────────╮ +# │ classes │ +# ╰─────────╯ + + class Cache(Object): """YouTube local cache.""" @@ -79,7 +84,7 @@ class Video(Object): :param d: video info :type d: dict """ - self.datetime = datetime.now().strftime("%Y%m%d%H%M%S") + self.at = datetime.now().strftime("%Y%m%d%H%M%S") self.uid = d["id"] self.fulltitle = d["fulltitle"] self.duration = d["duration"] @@ -136,7 +141,7 @@ def download_video(video_id: str | None) -> None: # ╰─────────╯ -def extract(opt: dict[str, Any], url: str) -> dict[str, Any]: +def extract(url: str) -> dict[str, Any]: """Return extracted dict. :rtype: dict @@ -159,7 +164,7 @@ def extract_playlist(playlist_id: str) -> dict: :type playlist_id: str :rtype: dict """ - return extract({}, url_playlist(playlist_id)) + return extract(url_playlist(playlist_id)) def extract_playlists(channel_id: str) -> dict: @@ -169,7 +174,7 @@ def extract_playlists(channel_id: str) -> dict: :type channel_id: str :rtype: dict """ - return extract({}, url_playlists(channel_id)) + return extract(url_playlists(channel_id)) def extract_video(video_id: str) -> dict: @@ -179,7 +184,7 @@ def extract_video(video_id: str) -> dict: :type video_id: str :rtype: dict """ - return extract({}, url_video(video_id)) + return extract(url_video(video_id)) def extract_videos(channel_id: str) -> dict: @@ -189,7 +194,7 @@ def extract_videos(channel_id: str) -> dict: :type channel_id: str :rtype: dict """ - return extract({}, url_videos(channel_id)) + return extract(url_videos(channel_id)) # ╭────────╮ From 3e04daeb8b8720127fd04333ca0f7682b53b257a Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 18:29:48 +0100 Subject: [PATCH 18/57] useless --- rwx/sw/ytdlp/__init__.py | 48 +++++----------------------------------- 1 file changed, 5 insertions(+), 43 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 326d5b1..378e436 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -85,6 +85,11 @@ class Video(Object): :type d: dict """ self.at = datetime.now().strftime("%Y%m%d%H%M%S") + # title + # description truncated + # duration + # thumbnails + # view_count self.uid = d["id"] self.fulltitle = d["fulltitle"] self.duration = d["duration"] @@ -93,22 +98,6 @@ class Video(Object): self.description = d["description"] -# channel -# title -# channel_follower_count -# description -# tags -# thumbnails -# uploader_id -# uploader -# videos / entries -# title -# description truncated -# duration -# thumbnails -# view_count - - # ╭──────────╮ # │ download │ # ╰──────────╯ @@ -197,33 +186,6 @@ def extract_videos(channel_id: str) -> dict: return extract(url_videos(channel_id)) -# ╭────────╮ -# │ filter │ -# ╰────────╯ - - -def filter_video(d: dict) -> dict: - """Return filtered video dict. - - :param d: video info dict - :type d: dict - :rtype: dict - """ - return { - "title": d["title"], - } - - -def filter_videos(d: dict) -> list[str]: - """Return filtered videos dict. - - :param d: videos dict - :type d: dict - :rtype: dict - """ - return [entry["id"] for entry in reversed(d["entries"])] - - # ╭──────╮ # │ next │ # ╰──────╯ From 979aaf129946285c10ab188ff8efc032b166cfd3 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 18:32:02 +0100 Subject: [PATCH 19/57] fix --- rwx/sw/ytdlp/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 378e436..b1fbff5 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -137,7 +137,6 @@ def extract(url: str) -> dict[str, Any]: """ d = ytdl( { - **opt, "extract_flat": True, "skip_download": True, }, From f73667a5d36713c0c48049ac049126a2dd851e60 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 20:22:11 +0100 Subject: [PATCH 20/57] wip --- rwx/sw/ytdlp/__init__.py | 65 +++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index b1fbff5..c947c88 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -44,35 +44,26 @@ class Channel(Object): self.uploader_id = d["uploader_id"] self.uploader = d["uploader"] # videos - self.videos_ids = [video["id"] for video in reversed(d["entries"])] - # TODO filter members-only + self.videos_ids = [ + entry["id"] + for entry in reversed(d["entries"]) + if entry["availability"] != "subscriber_only" + ] # playlists d = extract_playlists(channel_id) self.playlists_ids = [playlist["id"] for playlist in reversed(d["entries"])] - def add_video(self, d: dict) -> dict: - """Add video extra info.""" - self.video_index += 1 - log.info(f"{self.video_index} ∕ {len(self.videos_ids)}") - self.videos.append(Video(d)) - return d - def load_videos(self) -> None: """Load videos extra info.""" self.videos = [] - # a - #for index, video_id in enumerate(self.videos_ids): - # log.info(f"{index} ∕ {len(self.videos_ids)}") - # self.videos.append(Video(video_id)) - # b - videos_urls = [url_video(video_id) for video_id in self.videos_ids] - y = ytdl( - { - "process_info_hooks": [self.add_video], - "skip_download": True, - }, - ) - y.download(videos_urls) + for index, video_id in enumerate(self.videos_ids): + log.info(f"{index} ∕ {len(self.videos_ids)}") + self.videos.append(Video(video_id)) + + +# TODO Format +# TODO Playlist/basic,extra +# TODO Thumbnail class Video(Object): @@ -85,17 +76,31 @@ class Video(Object): :type d: dict """ self.at = datetime.now().strftime("%Y%m%d%H%M%S") - # title - # description truncated - # duration - # thumbnails - # view_count + # info self.uid = d["id"] - self.fulltitle = d["fulltitle"] - self.duration = d["duration"] + self.title = d["title"] + self.description_cut = d["description"] + self.duration = int(d["duration"]) + # TODO thumbnail from thumbnails + + def load_extra(self): + d = extract_video(self.uid) + # TODO formats + # TODO thumbnail from thumbnails + # TODO compare existing thumbnail + self.description = d["description"] + # TODO channel_id + self.duration = int(d["duration"]) + self.views = int(d["view_count"]) self.categories = d["categories"] self.tags = d["tags"] - self.description = d["description"] + # TODO automatic_captions + # TODO subtitles + self.chapters = d["chapters"] + self.likes = d["like_count"] + # TODO from epoch + self.timestamp = d["timestamp"] + self.fulltitle = d["fulltitle"] # ╭──────────╮ From 4fd1a87f7a0fb849fcacbea6dce43525b9f163fe Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 20:26:26 +0100 Subject: [PATCH 21/57] timestamp --- rwx/sw/ytdlp/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index c947c88..b947254 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -98,8 +98,7 @@ class Video(Object): # TODO subtitles self.chapters = d["chapters"] self.likes = d["like_count"] - # TODO from epoch - self.timestamp = d["timestamp"] + self.timestamp = datetime.fromtimestamp(d["timestamp"]).strftime("%Y%m%d%H%M%S") self.fulltitle = d["fulltitle"] From a1584c4f2e1755f042e0e6faad094f818b35d6f2 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 20:51:27 +0100 Subject: [PATCH 22/57] wip --- rwx/sw/ytdlp/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index b947254..031c779 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -9,6 +9,7 @@ from yt_dlp import YoutubeDL from rwx import Object from rwx.log import stream as log +TIMESTAMP = "%Y%m%d%H%M%S" URL = "https://youtube.com" @@ -41,11 +42,13 @@ class Channel(Object): self.description = d["description"] self.tags = d["tags"] # TODO thumbnails + self.thumbnails = d["thumbnails"] + # TODO thumbnail from thumbnails self.uploader_id = d["uploader_id"] self.uploader = d["uploader"] # videos - self.videos_ids = [ - entry["id"] + self.videos = [ + Video(entry) for entry in reversed(d["entries"]) if entry["availability"] != "subscriber_only" ] @@ -75,15 +78,15 @@ class Video(Object): :param d: video info :type d: dict """ - self.at = datetime.now().strftime("%Y%m%d%H%M%S") - # info self.uid = d["id"] self.title = d["title"] self.description_cut = d["description"] self.duration = int(d["duration"]) + self.thumbnails = [thumbnail["url"] for thumbnail in d["thumbnails"]] # TODO thumbnail from thumbnails def load_extra(self): + self.at = datetime.now().strftime(TIMESTAMP) d = extract_video(self.uid) # TODO formats # TODO thumbnail from thumbnails @@ -98,7 +101,7 @@ class Video(Object): # TODO subtitles self.chapters = d["chapters"] self.likes = d["like_count"] - self.timestamp = datetime.fromtimestamp(d["timestamp"]).strftime("%Y%m%d%H%M%S") + self.timestamp = datetime.fromtimestamp(d["timestamp"]).strftime(TIMESTAMP) self.fulltitle = d["fulltitle"] From f180a075640d3334b6bede7833ff6571c69d7ee1 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 20:57:37 +0100 Subject: [PATCH 23/57] thumbs --- rwx/sw/ytdlp/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 031c779..14ec62a 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -41,9 +41,8 @@ class Channel(Object): self.followers = int(d["channel_follower_count"]) self.description = d["description"] self.tags = d["tags"] - # TODO thumbnails - self.thumbnails = d["thumbnails"] - # TODO thumbnail from thumbnails + self.thumbnails = [thumbnail["url"] for thumbnail in d["thumbnails"]] + self.thumbnail = self.thumbnails[-1] self.uploader_id = d["uploader_id"] self.uploader = d["uploader"] # videos @@ -83,7 +82,7 @@ class Video(Object): self.description_cut = d["description"] self.duration = int(d["duration"]) self.thumbnails = [thumbnail["url"] for thumbnail in d["thumbnails"]] - # TODO thumbnail from thumbnails + self.thumbnail = self.thumbnails[-1] def load_extra(self): self.at = datetime.now().strftime(TIMESTAMP) From 35ab34c67d2ea4a2a0697721393e2940536bf169 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 21:09:38 +0100 Subject: [PATCH 24/57] thumb --- rwx/sw/ytdlp/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 14ec62a..4ad8504 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -81,14 +81,13 @@ class Video(Object): self.title = d["title"] self.description_cut = d["description"] self.duration = int(d["duration"]) - self.thumbnails = [thumbnail["url"] for thumbnail in d["thumbnails"]] - self.thumbnail = self.thumbnails[-1] + self.thumbnail = d["thumbnails"][-1]["url"] def load_extra(self): self.at = datetime.now().strftime(TIMESTAMP) d = extract_video(self.uid) # TODO formats - # TODO thumbnail from thumbnails + thumbnail = d["thumbnails"][-1]["url"] # TODO compare existing thumbnail self.description = d["description"] # TODO channel_id From 5a40282ac3611c7a01bc3213b986cc407b9bcbec Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 22:19:09 +0100 Subject: [PATCH 25/57] channel --- rwx/sw/ytdlp/__init__.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 4ad8504..7843a25 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -55,13 +55,6 @@ class Channel(Object): d = extract_playlists(channel_id) self.playlists_ids = [playlist["id"] for playlist in reversed(d["entries"])] - def load_videos(self) -> None: - """Load videos extra info.""" - self.videos = [] - for index, video_id in enumerate(self.videos_ids): - log.info(f"{index} ∕ {len(self.videos_ids)}") - self.videos.append(Video(video_id)) - # TODO Format # TODO Playlist/basic,extra @@ -87,10 +80,11 @@ class Video(Object): self.at = datetime.now().strftime(TIMESTAMP) d = extract_video(self.uid) # TODO formats + self.formats = d["formats"] thumbnail = d["thumbnails"][-1]["url"] # TODO compare existing thumbnail self.description = d["description"] - # TODO channel_id + self.channel_id = d["channel_id"] self.duration = int(d["duration"]) self.views = int(d["view_count"]) self.categories = d["categories"] From 49dd4728f63985f3a41e4748bafcfcc1fd2e725d Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 22:48:25 +0100 Subject: [PATCH 26/57] pl --- rwx/sw/ytdlp/__init__.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 7843a25..a89171a 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -53,11 +53,26 @@ class Channel(Object): ] # playlists d = extract_playlists(channel_id) - self.playlists_ids = [playlist["id"] for playlist in reversed(d["entries"])] + self.playlists = [Playlist(entry) for entry in reversed(d["entries"])] # TODO Format -# TODO Playlist/basic,extra + + +# TODO Playlist/extra +class Playlist(Object): + """YouTube playlist.""" + + def __init__(self, d: dict) -> None: + """Set playlist info. + + :param d: playlist info + :type d: dict + """ + self.uid = d["id"] + self.title = d["title"] + + # TODO Thumbnail @@ -140,7 +155,7 @@ def extract(url: str) -> dict[str, Any]: "skip_download": True, }, ).extract_info(url, download=False) - log.info(d) + log.debug(d) return d From e43409d40c4570be7d56ee8b434fd78ab0d1bea0 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Tue, 18 Mar 2025 23:41:04 +0100 Subject: [PATCH 27/57] toml --- rwx/sw/ytdlp/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index a89171a..69a5ede 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -7,6 +7,7 @@ from typing import Any from yt_dlp import YoutubeDL from rwx import Object +from rwx.fs import read_file_dict from rwx.log import stream as log TIMESTAMP = "%Y%m%d%H%M%S" @@ -21,8 +22,14 @@ URL = "https://youtube.com" class Cache(Object): """YouTube local cache.""" - def __init__(self, root: Path) -> None: - self.root = root + def __init__(self, root_file: Path) -> None: + self.root_file = root_file.resolve() + self.root_directory = self.root_file.parent + self.load() + + def load(self) -> None: + d = read_file_dict(self.root_file) + log.info(d) class Channel(Object): From a23e0eb12cb620b32be19c569d6ee0660f4b9d05 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Wed, 19 Mar 2025 12:09:01 +0100 Subject: [PATCH 28/57] yaml --- rwx/fs/__init__.py | 14 ++++++++++++++ rwx/sw/ytdlp/__init__.py | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/rwx/fs/__init__.py b/rwx/fs/__init__.py index 1873da2..b55713a 100644 --- a/rwx/fs/__init__.py +++ b/rwx/fs/__init__.py @@ -3,6 +3,7 @@ import os import shutil import tomllib +import yaml from pathlib import Path from rwx import ps @@ -132,6 +133,19 @@ def read_file_text(file_path: Path, charset: str = CHARSET) -> str: return read_file_bytes(file_path).decode(charset) +def read_file_yaml(file_path: Path, charset: str = CHARSET) -> dict | list: + """Read whole file as yaml object. + + :param file_path: source input file + :type file_path: Path + :param charset: charset to use for decoding input + :type charset: str + :rtype: dict + """ + text = read_file_text(file_path, charset) + return yaml.safe_load(text) + + def wipe(path: Path) -> None: """Wipe provided path, whether directory or file. diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 69a5ede..29a4305 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -7,7 +7,7 @@ from typing import Any from yt_dlp import YoutubeDL from rwx import Object -from rwx.fs import read_file_dict +from rwx.fs import read_file_yaml from rwx.log import stream as log TIMESTAMP = "%Y%m%d%H%M%S" @@ -28,7 +28,7 @@ class Cache(Object): self.load() def load(self) -> None: - d = read_file_dict(self.root_file) + d = read_file_yaml(self.root_file) log.info(d) From aaeb9e6e47bb8d1d4ccd38fabb47c44e5b1babfc Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Wed, 19 Mar 2025 12:20:08 +0100 Subject: [PATCH 29/57] webm --- rwx/sw/ytdlp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 29a4305..68ea71a 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -128,7 +128,7 @@ def download_video(video_id: str | None) -> None: if video_id: ytdl( { - "format": "bestvideo[ext=mp4]+bestaudio[ext=mp4]", + "format": "bestvideo[ext=webm]+bestaudio[ext=webm]", "outtmpl": "%(id)s.%(ext)s", "postprocessors": [ { From b7e4add332fc72fb88fe4d87f4c9ebd96bb81a4e Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Wed, 19 Mar 2025 12:31:56 +0100 Subject: [PATCH 30/57] ext --- rwx/sw/ytdlp/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 68ea71a..41750aa 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -10,6 +10,7 @@ from rwx import Object from rwx.fs import read_file_yaml from rwx.log import stream as log +EXT = "webm" TIMESTAMP = "%Y%m%d%H%M%S" URL = "https://youtube.com" @@ -128,7 +129,7 @@ def download_video(video_id: str | None) -> None: if video_id: ytdl( { - "format": "bestvideo[ext=webm]+bestaudio[ext=webm]", + "format": "+".join([f"best{av}[ext={EXT}]" for av in ["video", "audio"]]), "outtmpl": "%(id)s.%(ext)s", "postprocessors": [ { From 7ea1f10c8993846a390f193f9659460cf08019c4 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Wed, 19 Mar 2025 13:30:16 +0100 Subject: [PATCH 31/57] formats --- rwx/sw/ytdlp/__init__.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 41750aa..5c500b5 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -65,6 +65,32 @@ class Channel(Object): # TODO Format +class Format(Object): + """YouTube format.""" + + def __init__(self, d: dict) -> None: + """Set format info. + + :param d: format info + :type d: dict + """ + self.asr = d["asr"] + self.filesize = int(d["filesize"]) + self.format_id = d["format_id"] + self.format_note = d["format_id"] + self.fps = d["fps"] + self.height = int(d["height"]) + self.quality = int(d["quality"]) + self.width = int(d["width"]) + self.language = d["language"] + self.ext = d["ext"] + self.vcodec = d["vcodec"] + self.acodec = d["acodec"] + self.dynamic_range = d["dynamic_range"] + self.video_ext = d["video_ext"] + self.audio_ext = d["audio_ext"] + self.abr = d["abr"] + self.vbr = d["vbr"] # TODO Playlist/extra @@ -102,8 +128,7 @@ class Video(Object): def load_extra(self): self.at = datetime.now().strftime(TIMESTAMP) d = extract_video(self.uid) - # TODO formats - self.formats = d["formats"] + self.formats = [Format(format) for format in d["formats"]] thumbnail = d["thumbnails"][-1]["url"] # TODO compare existing thumbnail self.description = d["description"] From c3fad738b31be0ed0d538263d9a9350e036157a3 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Wed, 19 Mar 2025 17:50:15 +0100 Subject: [PATCH 32/57] wip --- rwx/sw/ytdlp/__init__.py | 53 ++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 5c500b5..5ada933 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -68,29 +68,41 @@ class Channel(Object): class Format(Object): """YouTube format.""" + @staticmethod + def get(d: dict, key: str) -> str | None: + value = d.get(key) + match value: + case "none": + return None + case _: + return value + def __init__(self, d: dict) -> None: """Set format info. :param d: format info :type d: dict """ - self.asr = d["asr"] - self.filesize = int(d["filesize"]) self.format_id = d["format_id"] - self.format_note = d["format_id"] - self.fps = d["fps"] - self.height = int(d["height"]) - self.quality = int(d["quality"]) - self.width = int(d["width"]) - self.language = d["language"] + self.format_note = d.get("format_note") + self.quality = d.get("quality") + self.language = d.get("language") self.ext = d["ext"] - self.vcodec = d["vcodec"] - self.acodec = d["acodec"] - self.dynamic_range = d["dynamic_range"] - self.video_ext = d["video_ext"] - self.audio_ext = d["audio_ext"] - self.abr = d["abr"] - self.vbr = d["vbr"] + # video + self.video_codec = Format.get(d, "vcodec") + if self.video_codec: + self.video_dynamic_range = d["dynamic_range"] + self.video_fps = d["fps"] + self.video_height = int(d["height"]) + self.video_bit_rate = d["vbr"] + self.video_ext = d["video_ext"] + self.video_width = int(d["width"]) + # audio + self.audio_codec = Format.get(d, "acodec") + if self.audio_codec: + self.audio_bit_rate = d["abr"] + self.audio_sampling_rate = d["asr"] + self.audio_ext = d["audio_ext"] # TODO Playlist/extra @@ -128,7 +140,16 @@ class Video(Object): def load_extra(self): self.at = datetime.now().strftime(TIMESTAMP) d = extract_video(self.uid) - self.formats = [Format(format) for format in d["formats"]] + self.audio_formats = [] + self.video_formats = [] + for entry in d["formats"]: + f = Format(entry) + if f.video_codec: + self.video_format = f + self.video_formats.append(f) + elif f.audio_codec: + self.audio_format = f + self.audio_formats.append(f) thumbnail = d["thumbnails"][-1]["url"] # TODO compare existing thumbnail self.description = d["description"] From 59ee8ed161dea20edb06321a775f8a5f22d1fc06 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Wed, 19 Mar 2025 17:58:29 +0100 Subject: [PATCH 33/57] lint --- rwx/sw/ytdlp/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 5ada933..b609019 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -162,7 +162,9 @@ class Video(Object): # TODO subtitles self.chapters = d["chapters"] self.likes = d["like_count"] - self.timestamp = datetime.fromtimestamp(d["timestamp"]).strftime(TIMESTAMP) + self.timestamp = datetime.fromtimestamp(d["timestamp"]).strftime( + TIMESTAMP + ) self.fulltitle = d["fulltitle"] @@ -175,7 +177,9 @@ def download_video(video_id: str | None) -> None: if video_id: ytdl( { - "format": "+".join([f"best{av}[ext={EXT}]" for av in ["video", "audio"]]), + "format": "+".join( + [f"best{av}[ext={EXT}]" for av in ["video", "audio"]] + ), "outtmpl": "%(id)s.%(ext)s", "postprocessors": [ { From b4dfb3e597f13f550d01d1fb4c2f65c20daed8d4 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Wed, 19 Mar 2025 19:52:04 +0100 Subject: [PATCH 34/57] wip --- rwx/sw/ytdlp/__init__.py | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index b609019..5f6af1f 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -83,26 +83,42 @@ class Format(Object): :param d: format info :type d: dict """ - self.format_id = d["format_id"] - self.format_note = d.get("format_note") - self.quality = d.get("quality") + self.uid = d["format_id"] + self.extension = d["ext"] + self.filesize = d.get("filesize") + self.filesize_approx = d.get("filesize_approx") self.language = d.get("language") - self.ext = d["ext"] + self.quality = d.get("quality") # video self.video_codec = Format.get(d, "vcodec") if self.video_codec: + self.video_bit_rate = d["vbr"] self.video_dynamic_range = d["dynamic_range"] + self.video_extension = d["video_ext"] self.video_fps = d["fps"] self.video_height = int(d["height"]) - self.video_bit_rate = d["vbr"] - self.video_ext = d["video_ext"] self.video_width = int(d["width"]) + else: + del self.video_codec # audio self.audio_codec = Format.get(d, "acodec") if self.audio_codec: self.audio_bit_rate = d["abr"] + self.audio_channels = int(d["audio_channels"]) + self.audio_extension = d["audio_ext"] self.audio_sampling_rate = d["asr"] - self.audio_ext = d["audio_ext"] + else: + del self.audio_codec + + def audio(self) -> str: + return f"{self.uid} \ +→ {self.audio_sampling_rate} × {self.audio_channels} \ +@ {self.audio_bit_rate} × {self.audio_codec}" + + def video(self) -> str: + return f"{self.uid} \ +→ {self.video_width} × {self.video_height} × {self.video_fps} \ +@ {self.video_bit_rate} × {self.video_codec}" # TODO Playlist/extra @@ -144,10 +160,10 @@ class Video(Object): self.video_formats = [] for entry in d["formats"]: f = Format(entry) - if f.video_codec: + if hasattr(f, "video_codec"): self.video_format = f self.video_formats.append(f) - elif f.audio_codec: + elif hasattr(f, "audio_codec"): self.audio_format = f self.audio_formats.append(f) thumbnail = d["thumbnails"][-1]["url"] @@ -177,9 +193,7 @@ def download_video(video_id: str | None) -> None: if video_id: ytdl( { - "format": "+".join( - [f"best{av}[ext={EXT}]" for av in ["video", "audio"]] - ), + "format": "bestvideo[ext=webm]+bestaudio[ext=webm]", "outtmpl": "%(id)s.%(ext)s", "postprocessors": [ { From 12ac6cad399a0a9b25e294b8fffcd99313a0ff3b Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Wed, 19 Mar 2025 21:12:46 +0100 Subject: [PATCH 35/57] subs --- rwx/sw/ytdlp/__init__.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 5f6af1f..99d93e9 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -10,7 +10,7 @@ from rwx import Object from rwx.fs import read_file_yaml from rwx.log import stream as log -EXT = "webm" +SUBTITLES_EXTENSIONS = ["vtt"] TIMESTAMP = "%Y%m%d%H%M%S" URL = "https://youtube.com" @@ -135,6 +135,20 @@ class Playlist(Object): self.title = d["title"] +class Subtitles(Object): + """YouTube subtitles.""" + + def __init__(self, uid: str, d: dict) -> None: + """Set subtitles info. + + :param d: subtitles info + :type d: dict + """ + self.uid = uid + self.extension = d["ext"] + self.name = d["name"] + + # TODO Thumbnail @@ -175,7 +189,12 @@ class Video(Object): self.categories = d["categories"] self.tags = d["tags"] # TODO automatic_captions - # TODO subtitles + self.subtitles = [] + for uid, entries in d["subtitles"].items(): + for entry in entries: + subtitles = Subtitles(uid, entry) + if subtitles.extension in SUBTITLES_EXTENSIONS: + self.subtitles.append(subtitles) self.chapters = d["chapters"] self.likes = d["like_count"] self.timestamp = datetime.fromtimestamp(d["timestamp"]).strftime( From 1cf50619fd1f0f201ce9fce8012b22f72f370a08 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Wed, 19 Mar 2025 21:44:07 +0100 Subject: [PATCH 36/57] captions --- rwx/sw/ytdlp/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 99d93e9..ebe39f0 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -188,7 +188,12 @@ class Video(Object): self.views = int(d["view_count"]) self.categories = d["categories"] self.tags = d["tags"] - # TODO automatic_captions + self.automatic_captions = [] + for uid, entries in d["automatic_captions"].items(): + for entry in entries: + subtitles = Subtitles(uid, entry) + if subtitles.extension in SUBTITLES_EXTENSIONS: + self.automatic_captions.append(subtitles) self.subtitles = [] for uid, entries in d["subtitles"].items(): for entry in entries: From cd3a584ef1b1eb0b5feaf98002c74d50e80883c9 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Wed, 19 Mar 2025 22:02:22 +0100 Subject: [PATCH 37/57] url --- rwx/sw/ytdlp/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index ebe39f0..447e537 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -147,6 +147,7 @@ class Subtitles(Object): self.uid = uid self.extension = d["ext"] self.name = d["name"] + self.url = d["url"] # TODO Thumbnail From fd4184c6a816b291cf25d7b1e8ef201cebd4288d Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Wed, 19 Mar 2025 22:03:40 +0100 Subject: [PATCH 38/57] descut --- rwx/sw/ytdlp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rwx/sw/ytdlp/__init__.py b/rwx/sw/ytdlp/__init__.py index 447e537..9045781 100644 --- a/rwx/sw/ytdlp/__init__.py +++ b/rwx/sw/ytdlp/__init__.py @@ -162,9 +162,9 @@ class Video(Object): :param d: video info :type d: dict """ + self.description_cut = d["description"] self.uid = d["id"] self.title = d["title"] - self.description_cut = d["description"] self.duration = int(d["duration"]) self.thumbnail = d["thumbnails"][-1]["url"] From f4136fd5a9ad6b1a81e456417632bcefee72f972 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 30 Mar 2025 12:18:16 +0200 Subject: [PATCH 39/57] cs/draft --- sh/cryptsetup.sh | 52 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/sh/cryptsetup.sh b/sh/cryptsetup.sh index 900083d..c258005 100644 --- a/sh/cryptsetup.sh +++ b/sh/cryptsetup.sh @@ -1,6 +1,52 @@ -_rwx_cmd_cs() { rwx_crypt_setup "${@}"; } +_rwx_cmd_cs() { rwx_crypt "${@}"; } -rwx_crypt_setup() { +RWX_CRYPT_ROOT="${HOME}/home/crypt" + +rwx_crypt() { local action="${1}" - echo "cs: ${action}" + local action_close="close" + local action_open="open" + local mapper="/dev/mapper" + local mount_root="/media" + local crypt_arg crypt_file crypt_map crypt_mount pass_phrase + case "${action}" in + "${action_close}" | "${action_open}") + shift + if [ -z "${1}" ]; then + rwx_log_error 1 "No files" + fi + [ "${action}" = "${action_open}" ] && + pass_phrase="$(rwx_read_passphrase)" + for crypt_arg in "${@}"; do + rwx_log_info + crypt_file="${RWX_CRYPT_ROOT}/${crypt_arg}" + if [ -f "${crypt_file}" ]; then + crypt_map="${mapper}/${crypt_arg}" + crypt_mount="${mount_root}/${crypt_arg}" + case "${action}" in + "${action_close}") + rwx_log_info "CLOSE" + # TODO unmount file system + # TODO close device + # TODO disconnect device + ;; + "${action_open}") + rwx_log_info "OPEN" + # TODO find next available device + # TODO connect device + # TODO open device + # TODO mount file system + ;; + *) ;; + esac + else + rwx_log_error 2 "Not a file" + fi + done + ;; + *) + rwx_log_info "Usage:" + rwx_log_info "${action_close}|${action_open}" + ;; + esac } From bb894045130b6b13cf0fbd41d047c2eadf2a61a8 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 30 Mar 2025 12:41:48 +0200 Subject: [PATCH 40/57] =?UTF-8?q?=E2=86=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sh/cryptsetup.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sh/cryptsetup.sh b/sh/cryptsetup.sh index c258005..545ca67 100644 --- a/sh/cryptsetup.sh +++ b/sh/cryptsetup.sh @@ -24,12 +24,6 @@ rwx_crypt() { crypt_map="${mapper}/${crypt_arg}" crypt_mount="${mount_root}/${crypt_arg}" case "${action}" in - "${action_close}") - rwx_log_info "CLOSE" - # TODO unmount file system - # TODO close device - # TODO disconnect device - ;; "${action_open}") rwx_log_info "OPEN" # TODO find next available device @@ -37,6 +31,12 @@ rwx_crypt() { # TODO open device # TODO mount file system ;; + "${action_close}") + rwx_log_info "CLOSE" + # TODO unmount file system + # TODO close device + # TODO disconnect device + ;; *) ;; esac else @@ -47,6 +47,7 @@ rwx_crypt() { *) rwx_log_info "Usage:" rwx_log_info "${action_close}|${action_open}" + # TODO list ;; esac } From c6f6d42af6a25660028fc2fb1848dc030c767289 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 30 Mar 2025 12:43:20 +0200 Subject: [PATCH 41/57] dirs --- sh/cryptsetup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sh/cryptsetup.sh b/sh/cryptsetup.sh index 545ca67..c76735e 100644 --- a/sh/cryptsetup.sh +++ b/sh/cryptsetup.sh @@ -29,11 +29,13 @@ rwx_crypt() { # TODO find next available device # TODO connect device # TODO open device + # TODO make mount directory # TODO mount file system ;; "${action_close}") rwx_log_info "CLOSE" # TODO unmount file system + # TODO remove mount directory # TODO close device # TODO disconnect device ;; From e0ba419f2abc52ad4359964796f5c11c60c9168d Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 30 Mar 2025 12:53:39 +0200 Subject: [PATCH 42/57] cs/root --- sh/cryptsetup.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/sh/cryptsetup.sh b/sh/cryptsetup.sh index c76735e..0f2629c 100644 --- a/sh/cryptsetup.sh +++ b/sh/cryptsetup.sh @@ -4,6 +4,7 @@ RWX_CRYPT_ROOT="${HOME}/home/crypt" rwx_crypt() { local action="${1}" + shift local action_close="close" local action_open="open" local mapper="/dev/mapper" @@ -11,10 +12,12 @@ rwx_crypt() { local crypt_arg crypt_file crypt_map crypt_mount pass_phrase case "${action}" in "${action_close}" | "${action_open}") - shift - if [ -z "${1}" ]; then - rwx_log_error 1 "No files" - fi + local user_id + user_id="$(id --user)" + [ "${user_id}" -eq 0 ] || + rwx_log_error 1 "Not root" + [ -n "${1}" ] || + rwx_log_error 2 "No files" [ "${action}" = "${action_open}" ] && pass_phrase="$(rwx_read_passphrase)" for crypt_arg in "${@}"; do @@ -29,6 +32,7 @@ rwx_crypt() { # TODO find next available device # TODO connect device # TODO open device + echo "${pass_phrase}" # TODO make mount directory # TODO mount file system ;; @@ -42,7 +46,7 @@ rwx_crypt() { *) ;; esac else - rwx_log_error 2 "Not a file" + rwx_log_error 3 "Not a file" fi done ;; From f4f1aeaccff2253721a474438fff37213d566a54 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 30 Mar 2025 16:45:48 +0200 Subject: [PATCH 43/57] cs/device --- sh/cryptsetup.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/sh/cryptsetup.sh b/sh/cryptsetup.sh index 0f2629c..b7da532 100644 --- a/sh/cryptsetup.sh +++ b/sh/cryptsetup.sh @@ -29,7 +29,19 @@ rwx_crypt() { case "${action}" in "${action_open}") rwx_log_info "OPEN" - # TODO find next available device + local nbd_device nbd_size + local nbd_index=0 + while [ -z "${nbd_device}" ]; do + if [ -f "/dev/nbd${nbd_index}" ]; then + nbd_size="$(cat /sys/block/nbd"${nbd_index}/size")" + [ "${nbd_size}" -eq 0 ] && + nbd_device="/dev/nbd${nbd_index}" + fi + nbd_index=$((nbd_index + 1)) + done + [ -z "${nbd_device}" ] && + rwx_log_error 4 "No device available" + rwx_log_info "device: ${nbd_device}" # TODO connect device # TODO open device echo "${pass_phrase}" From aa2ed7014b6075df0f773726fb65220a0bfd3e57 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 30 Mar 2025 18:50:51 +0200 Subject: [PATCH 44/57] crypt/device --- sh/cryptsetup.sh | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/sh/cryptsetup.sh b/sh/cryptsetup.sh index b7da532..a2d5c8b 100644 --- a/sh/cryptsetup.sh +++ b/sh/cryptsetup.sh @@ -2,9 +2,30 @@ _rwx_cmd_cs() { rwx_crypt "${@}"; } RWX_CRYPT_ROOT="${HOME}/home/crypt" +rwx_crypt_device() { + local device size + local index=0 + while [ -z "${device}" ]; do + device="/dev/nbd${index}" + if [ -b "${device}" ]; then + size="$(cat /sys/block/nbd"${index}/size")" + [ "${size}" -eq 0 ] || + device="" + else + device="" + break + fi + index=$((index + 1)) + done + if [ -n "${device}" ]; then + echo "${device}" + else + rwx_log_error 1 "No device available" + fi +} + rwx_crypt() { local action="${1}" - shift local action_close="close" local action_open="open" local mapper="/dev/mapper" @@ -12,6 +33,7 @@ rwx_crypt() { local crypt_arg crypt_file crypt_map crypt_mount pass_phrase case "${action}" in "${action_close}" | "${action_open}") + shift local user_id user_id="$(id --user)" [ "${user_id}" -eq 0 ] || @@ -22,34 +44,20 @@ rwx_crypt() { pass_phrase="$(rwx_read_passphrase)" for crypt_arg in "${@}"; do rwx_log_info - crypt_file="${RWX_CRYPT_ROOT}/${crypt_arg}" + crypt_file="${RWX_CRYPT_ROOT}/${crypt_arg}.qcow2" if [ -f "${crypt_file}" ]; then crypt_map="${mapper}/${crypt_arg}" crypt_mount="${mount_root}/${crypt_arg}" case "${action}" in "${action_open}") - rwx_log_info "OPEN" - local nbd_device nbd_size - local nbd_index=0 - while [ -z "${nbd_device}" ]; do - if [ -f "/dev/nbd${nbd_index}" ]; then - nbd_size="$(cat /sys/block/nbd"${nbd_index}/size")" - [ "${nbd_size}" -eq 0 ] && - nbd_device="/dev/nbd${nbd_index}" - fi - nbd_index=$((nbd_index + 1)) - done - [ -z "${nbd_device}" ] && - rwx_log_error 4 "No device available" - rwx_log_info "device: ${nbd_device}" + local nbd_device="$(rwx_crypt_device)" + echo "device: ${nbd_device}" # TODO connect device # TODO open device - echo "${pass_phrase}" # TODO make mount directory # TODO mount file system ;; "${action_close}") - rwx_log_info "CLOSE" # TODO unmount file system # TODO remove mount directory # TODO close device From 36a04798822f77d2c4534d92f05a3b3035b6977c Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 30 Mar 2025 20:18:20 +0200 Subject: [PATCH 45/57] crypt/open,close --- sh/cryptsetup.sh | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/sh/cryptsetup.sh b/sh/cryptsetup.sh index a2d5c8b..3927886 100644 --- a/sh/cryptsetup.sh +++ b/sh/cryptsetup.sh @@ -50,23 +50,47 @@ rwx_crypt() { crypt_mount="${mount_root}/${crypt_arg}" case "${action}" in "${action_open}") - local nbd_device="$(rwx_crypt_device)" - echo "device: ${nbd_device}" - # TODO connect device - # TODO open device - # TODO make mount directory - # TODO mount file system + local device + if ! device="$(rwx_crypt_device)"; then + rwx_log_error 4 "No device available" + fi + # connect device + if ! qemu-nbd --connect "${device}" "${crypt_file}"; then + rwx_log_error 5 "Connection failure: ${device}" + fi + # open device + echo "${pass_phrase}" | + cryptsetup luksOpen "${device}" "${crypt_arg}" + # make mount directory + if ! mkdir --parents "${crypt_mount}"; then + rwx_log_error 7 "Making failure: ${crypt_mount}" + fi + # mount file system + if ! mount \ + --options "autodefrag,compress-force=zstd" \ + "${crypt_map}" "${crypt_mount}"; then + rwx_log_error 8 "Mounting failure: ${crypt_map}" + fi ;; "${action_close}") - # TODO unmount file system - # TODO remove mount directory - # TODO close device + # unmount file system + if ! umount "${crypt_mount}"; then + rwx_log_error 4 "Unmounting failure: ${crypt_mount}" + fi + # remove mount directory + if ! rmdir "${crypt_mount}"; then + rwx_log_error 5 "Removal failure: ${crypt_mount}" + fi + # close device + if ! cryptsetup luksClose "${crypt_arg}"; then + rwx_log_error 6 "Closing failure: ${crypt_arg}" + fi # TODO disconnect device ;; *) ;; esac else - rwx_log_error 3 "Not a file" + rwx_log_error 3 "Not a file: ${crypt_file}" fi done ;; From d034a99da7f50a84e656c2c37853021d7812e0ba Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 30 Mar 2025 20:19:58 +0200 Subject: [PATCH 46/57] crypt/disconnect --- sh/cryptsetup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/sh/cryptsetup.sh b/sh/cryptsetup.sh index 3927886..19c4232 100644 --- a/sh/cryptsetup.sh +++ b/sh/cryptsetup.sh @@ -86,6 +86,7 @@ rwx_crypt() { rwx_log_error 6 "Closing failure: ${crypt_arg}" fi # TODO disconnect device + rwx_log_error 7 "Disconnecting failure: ${crypt_arg}" ;; *) ;; esac From 8d869993a7b87f21587247e36b882573915ae494 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 30 Mar 2025 21:03:06 +0200 Subject: [PATCH 47/57] cs/home --- sh/cryptsetup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sh/cryptsetup.sh b/sh/cryptsetup.sh index 19c4232..024a315 100644 --- a/sh/cryptsetup.sh +++ b/sh/cryptsetup.sh @@ -1,6 +1,6 @@ _rwx_cmd_cs() { rwx_crypt "${@}"; } -RWX_CRYPT_ROOT="${HOME}/home/crypt" +RWX_CRYPT_ROOT="/data/home/user/crypt" rwx_crypt_device() { local device size From 40d6d793948f610dcb8d310d6bfd531267bf64c1 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 30 Mar 2025 22:17:01 +0200 Subject: [PATCH 48/57] cs/wip --- sh/cryptsetup.sh | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/sh/cryptsetup.sh b/sh/cryptsetup.sh index 024a315..b562ce3 100644 --- a/sh/cryptsetup.sh +++ b/sh/cryptsetup.sh @@ -1,6 +1,7 @@ _rwx_cmd_cs() { rwx_crypt "${@}"; } RWX_CRYPT_ROOT="/data/home/user/crypt" +RWX_CRYPT_VAR="/var/lib/crypt" rwx_crypt_device() { local device size @@ -48,28 +49,35 @@ rwx_crypt() { if [ -f "${crypt_file}" ]; then crypt_map="${mapper}/${crypt_arg}" crypt_mount="${mount_root}/${crypt_arg}" + local device case "${action}" in "${action_open}") - local device if ! device="$(rwx_crypt_device)"; then rwx_log_error 4 "No device available" fi + # record device + if ! rwx_file_write \ + "${RWX_CRYPT_VAR}/${crypt_arg}" "${device}"; then + rwx_log_error 5 "Writing failure: ${device}" + fi # connect device if ! qemu-nbd --connect "${device}" "${crypt_file}"; then - rwx_log_error 5 "Connection failure: ${device}" + rwx_log_error 6 "Connection failure: ${device}" fi # open device - echo "${pass_phrase}" | - cryptsetup luksOpen "${device}" "${crypt_arg}" + if ! echo "${pass_phrase}" | + cryptsetup luksOpen "${device}" "${crypt_arg}"; then + rwx_log_error 7 "Opening failure: ${device}" + fi # make mount directory if ! mkdir --parents "${crypt_mount}"; then - rwx_log_error 7 "Making failure: ${crypt_mount}" + rwx_log_error 8 "Making failure: ${crypt_mount}" fi # mount file system if ! mount \ --options "autodefrag,compress-force=zstd" \ "${crypt_map}" "${crypt_mount}"; then - rwx_log_error 8 "Mounting failure: ${crypt_map}" + rwx_log_error 9 "Mounting failure: ${crypt_map}" fi ;; "${action_close}") @@ -85,8 +93,14 @@ rwx_crypt() { if ! cryptsetup luksClose "${crypt_arg}"; then rwx_log_error 6 "Closing failure: ${crypt_arg}" fi - # TODO disconnect device - rwx_log_error 7 "Disconnecting failure: ${crypt_arg}" + # load device + if ! device="$(cat "${RWX_CRYPT_ROOT}/${crypt_arg}")"; then + rwx_log_error 7 "Loading failure: ${crypt_arg}" + fi + # disconnect device + if ! qemu-nbd --disconnect "${device}"; then + rwx_log_error 8 "Disconnection failure: ${device}" + fi ;; *) ;; esac From 930f296dc812bbf579266c52242305172ddf4778 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 30 Mar 2025 22:18:56 +0200 Subject: [PATCH 49/57] cs/rm --- sh/cryptsetup.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sh/cryptsetup.sh b/sh/cryptsetup.sh index b562ce3..ba94a8a 100644 --- a/sh/cryptsetup.sh +++ b/sh/cryptsetup.sh @@ -101,6 +101,10 @@ rwx_crypt() { if ! qemu-nbd --disconnect "${device}"; then rwx_log_error 8 "Disconnection failure: ${device}" fi + # remove record + if ! rm "${RWX_CRYPT_ROOT}/${crypt_arg}"; then + rwx_log_error 9 "Removal failure: ${crypt_arg}" + fi ;; *) ;; esac From 715708dc3d2771d41b35c7a7aa478308ab862d76 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 30 Mar 2025 22:25:22 +0200 Subject: [PATCH 50/57] cs/mkdir --- sh/cryptsetup.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/sh/cryptsetup.sh b/sh/cryptsetup.sh index ba94a8a..a4792e6 100644 --- a/sh/cryptsetup.sh +++ b/sh/cryptsetup.sh @@ -52,32 +52,37 @@ rwx_crypt() { local device case "${action}" in "${action_open}") + # find device if ! device="$(rwx_crypt_device)"; then rwx_log_error 4 "No device available" fi + # make directory + if ! mkdir --parents "${RWX_CRYPT_VAR}"; then + rwx_log_error 5 "Making failure: ${RWX_CRYPT_VAR}" + fi # record device if ! rwx_file_write \ "${RWX_CRYPT_VAR}/${crypt_arg}" "${device}"; then - rwx_log_error 5 "Writing failure: ${device}" + rwx_log_error 6 "Writing failure: ${device}" fi # connect device if ! qemu-nbd --connect "${device}" "${crypt_file}"; then - rwx_log_error 6 "Connection failure: ${device}" + rwx_log_error 7 "Connection failure: ${device}" fi # open device if ! echo "${pass_phrase}" | cryptsetup luksOpen "${device}" "${crypt_arg}"; then - rwx_log_error 7 "Opening failure: ${device}" + rwx_log_error 8 "Opening failure: ${device}" fi # make mount directory if ! mkdir --parents "${crypt_mount}"; then - rwx_log_error 8 "Making failure: ${crypt_mount}" + rwx_log_error 9 "Making failure: ${crypt_mount}" fi # mount file system if ! mount \ --options "autodefrag,compress-force=zstd" \ "${crypt_map}" "${crypt_mount}"; then - rwx_log_error 9 "Mounting failure: ${crypt_map}" + rwx_log_error 10 "Mounting failure: ${crypt_map}" fi ;; "${action_close}") From a7219cf7faa4f32ae5aff9459854d21e93224d1c Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sun, 30 Mar 2025 22:31:42 +0200 Subject: [PATCH 51/57] cs/fix --- sh/cryptsetup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sh/cryptsetup.sh b/sh/cryptsetup.sh index a4792e6..fd20a64 100644 --- a/sh/cryptsetup.sh +++ b/sh/cryptsetup.sh @@ -99,7 +99,7 @@ rwx_crypt() { rwx_log_error 6 "Closing failure: ${crypt_arg}" fi # load device - if ! device="$(cat "${RWX_CRYPT_ROOT}/${crypt_arg}")"; then + if ! device="$(cat "${RWX_CRYPT_VAR}/${crypt_arg}")"; then rwx_log_error 7 "Loading failure: ${crypt_arg}" fi # disconnect device @@ -107,7 +107,7 @@ rwx_crypt() { rwx_log_error 8 "Disconnection failure: ${device}" fi # remove record - if ! rm "${RWX_CRYPT_ROOT}/${crypt_arg}"; then + if ! rm "${RWX_CRYPT_VAR}/${crypt_arg}"; then rwx_log_error 9 "Removal failure: ${crypt_arg}" fi ;; From 510371ef24a31fe1a8ca027f31b31b24cc0a8dcd Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Thu, 3 Apr 2025 10:55:52 +0200 Subject: [PATCH 52/57] glao,glo --- sh/alias/git.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/sh/alias/git.sh b/sh/alias/git.sh index cadd10b..fe5380a 100644 --- a/sh/alias/git.sh +++ b/sh/alias/git.sh @@ -307,6 +307,14 @@ a__git_log_all() { "${@}" } +# log all history as oneline +glao() { a__git_log_all_oneline "${@}"; } +a__git_log_all_oneline() { + a__git_log_all \ + --oneline \ + "${@}" +} + # log all history with patches glap() { a__git_log_all_patch "${@}"; } a__git_log_all_patch() { @@ -316,6 +324,14 @@ a__git_log_all_patch() { "${@}" } +# log history as oneline +glo() { a__git_log_oneline "${@}"; } +a__git_log_oneline() { + a__git_log \ + --oneline \ + "${@}" +} + # log history with patches glp() { a__git_log_patch "${@}"; } a__git_log_patch() { From ce64258fceb36b6e26dd46aa06ecb72089deab2b Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Mon, 12 May 2025 17:40:19 +0200 Subject: [PATCH 53/57] web/fetch --- rwx/web/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rwx/web/__init__.py b/rwx/web/__init__.py index 943a3f4..e746872 100644 --- a/rwx/web/__init__.py +++ b/rwx/web/__init__.py @@ -1,7 +1,15 @@ +import requests + from rwx import Object, txt from rwx.txt import CHARSET +def fetch(url: str) -> str: + response = requests.get(url) + response.raise_for_status() + return response.text + + class Page(Object): def __init__(self): self.charset = CHARSET From 5c3f75caa1bfdf26d646794e4c3da2cf2030c8b7 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sat, 17 May 2025 01:06:26 +0200 Subject: [PATCH 54/57] hdmi/yuv420p --- sh/ffmpeg.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sh/ffmpeg.sh b/sh/ffmpeg.sh index bcb3a62..cdee30e 100644 --- a/sh/ffmpeg.sh +++ b/sh/ffmpeg.sh @@ -72,8 +72,9 @@ rwx_ffmpeg_input_hdmi() { set -- \ -f "v4l2" \ -video_size "1920x1080" \ - -framerate "60" \ + -framerate "120" \ -input_format "yuyv422" \ + -pix_fmt "yuv420p" \ -i "${device}" local argument for argument in "${@}"; do echo "${argument}"; done From d1a3de55ab78a1e3d8bb793622f4e256a99334ea Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sat, 17 May 2025 01:19:27 +0200 Subject: [PATCH 55/57] hdmi/yuv420p --- sh/ffmpeg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sh/ffmpeg.sh b/sh/ffmpeg.sh index cdee30e..c32ad06 100644 --- a/sh/ffmpeg.sh +++ b/sh/ffmpeg.sh @@ -73,7 +73,7 @@ rwx_ffmpeg_input_hdmi() { -f "v4l2" \ -video_size "1920x1080" \ -framerate "120" \ - -input_format "yuyv422" \ + -input_format "yuv420p" \ -pix_fmt "yuv420p" \ -i "${device}" local argument From 5efa62dd403e360b04f17ee3a15f192feb401d57 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sat, 7 Jun 2025 23:11:01 +0200 Subject: [PATCH 56/57] =?UTF-8?q?build=E2=86=92render?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rwx/prj/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rwx/prj/__init__.py b/rwx/prj/__init__.py index ea6a67e..f545f73 100644 --- a/rwx/prj/__init__.py +++ b/rwx/prj/__init__.py @@ -22,4 +22,4 @@ class Project(Object): def build(self) -> None: """Build the project.""" - run(str(self.root / "build.py")) + run(str(self.root / "render.py")) From 7eae3bed9fd04c5730477d8e4c28b947e1142ae5 Mon Sep 17 00:00:00 2001 From: Marc Beninca Date: Sat, 7 Jun 2025 23:14:21 +0200 Subject: [PATCH 57/57] render --- build.py => render.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build.py => render.py (100%) diff --git a/build.py b/render.py similarity index 100% rename from build.py rename to render.py