1. Fix some video automatic_captions is empty (eg:VSFg5LZYsyc).

3. Support subtitle lang format like lang-code.
2. Add requested_subtitles and parse it first.

fetchYoutubeSubtitle.py

@@ -1,13 +1,17 @@
+import os
 import json
 import math
 import time
+import traceback
 from typing import Optional
 import xml.etree.ElementTree as ElementTree
 from html import unescape
 import yt_dlp
 
+debug = os.getenv("DEBUG")
 # yt-dlp subtitle types: json3,srv1,srv2,srv3,ttml,vtt, xml(youtube url with out extargs)
 
+
 # "subtitles": {
 #     "live_chat": [
 #       {
@@ -18,53 +22,134 @@ import yt_dlp
 #       }
 #     ]
 #   }
-def getUrlFromSubtitles(item, lang='en', subType="vtt"):
-    langs = item.keys()
-    if len(langs) == 0:
-        return None
-
-    l = lang if lang in langs else ('en' if 'en' in langs else list(langs)[0] )
-    if l is  None:
-        return
-
-    for subtitle in item[l]:
-        # print("getUrlFromSubtitles subtitle: %s" % subtitle)
-        if l != "live_chat" and subType =="xml":
-            # print("subtitle source url: {}".format(subtitle.get("url")))
-            return subtitle.get("url").replace("&fmt="+subtitle.get("ext"),"")
+def getUrlFromSubtitleItem(item, lang="en", subType="vtt"):
+    for subtitle in item[lang]:
+        if lang != "live_chat" and subType == "xml":
+            if debug:
+                print(
+                    "subtitle source lang:{} url: {}".format(lang, subtitle.get("url"))
+                )
+            return subtitle.get("url").replace("&fmt=" + subtitle.get("ext"), "")
         if subtitle.get("ext") == subType:
+            if debug:
+                print("subtitle lang:{} url: {}".format(lang, subtitle.get("url")))
             return subtitle.get("url")
+
+    return None
+
+
+def getRequestedSubtitlesUrl(info_dict, lang, subType):
+    item = info_dict.get("requested_subtitles")
+    langs = item.keys()
+    for l in langs:
+        if l.startswith(lang):
+            item = {l: [item[l]]} if type(item[l]) == dict else item
+            url = getUrlFromSubtitleItem(item, l, subType)
+            if url:
+                if debug:
+                    print("getRequestedSubtitlesUrl lang:{} url:{}".format(l, url))
+                return url
+    return None
+
+
+def getSubtitleLangUrl(
+    info_dict,
+    lang="en",
+    subType="vtt",
+    subTitleKeys=["subtitles", "automatic_captions"],
+):
+    for subtitle_item in subTitleKeys:
+        langs = info_dict.get(subtitle_item).keys()
+        if lang in langs:
+            url = getUrlFromSubtitleItem(info_dict.get(subtitle_item), lang, subType)
+            if url:
+                if debug:
+                    print("getSubtitleLangUrl lang:{}".format(lang))
+                return url
+
+    for subtitle_item in subTitleKeys:
+        langs = info_dict.get(subtitle_item).keys()
+        for l in langs:
+            if l.startswith(lang):
+                url = getUrlFromSubtitleItem(info_dict.get(subtitle_item), l, subType)
+                if url:
+                    if debug:
+                        print("getSubtitleLangUrl lang:{} url:{}".format(l, url))
+                    return url
+
+    return None
+
+
+def getSubtitleOtherUrl(
+    info_dict,
+    lang="en",
+    subType="vtt",
+    subTitleKeys=["subtitles", "automatic_captions"],
+):
+    for subtitle_item in subTitleKeys:
+        langs = info_dict.get(subtitle_item).keys()
+        if len(langs) == 0:
+            continue
+
+        l = lang if lang in langs else ("en" if "en" in langs else list(langs)[0])
+        if l is None:
+            continue
+
+        url = getUrlFromSubtitleItem(info_dict.get(subtitle_item), l, subType)
+        if url:
+            if debug:
+                print("getSubtitleOtherUrl lang:{} url:{}".format(l, url))
+            return url
+
     return None
 
-async def fetchSubtitle(url: str, lang: Optional[str] = 'en', subType: Optional[str] = "vtt", proxy: Optional[str] = None) -> dict:
+
+async def fetchSubtitle(
+    url: str,
+    lang: Optional[str] = "en",
+    subType: Optional[str] = "vtt",
+    proxy: Optional[str] = None,
+) -> dict:
     return await fetchSubtitlebyType(url, lang, subType, proxy)
 
-async def fetchSubtitlebyType(url: str, lang: Optional[str] = 'en', subType: Optional[str] = "vtt", proxy: Optional[str] = None) -> dict:
+
+async def fetchSubtitlebyType(
+    url: str,
+    lang: Optional[str] = "en",
+    subType: Optional[str] = "vtt",
+    proxy: Optional[str] = None,
+) -> dict:
+    # lang-code or lang.* .* is regex
+    reqLang = lang if len(lang.split("-")) > 1 or lang.endswith(".*") else lang + ".*"
+
     ydl_opts = {
         "noplaylist": True,
-        "writesubtitles": False,
-        "allsubtitles": True,
-        "subtitleslangs": [lang] if lang else [],
+        "writesubtitles": True,
+        "writeautomaticsub": True,
+        # "listsubtitles": True,
+        # "subtitlesformat": subType, # mark due to default youtube no srt and xml format
+        "subtitleslangs": [reqLang],
         "skip_download": True,
         "socket_timeout": 10,
         "extractor_retries": 0,
         "extractor_args": {
             "youtube": {
                 "player_skip": ["configs", "initial"],  # "webpage",
-                "player_client": ["android"],
-                "skip": ["hls", "dash", "translated_subs"],
+                "player_client": ["web"],
+                "skip": ["hls", "dash"],  # don't skip "translated_subs"
             }
         },
     }
 
     if proxy:
         ydl_opts.update({"proxy": proxy, "socket_timeout": 20})
-
+    # print(ydl_opts)
     title = "unknow"
     duration = ""
     try:
         with yt_dlp.YoutubeDL(ydl_opts) as ydl:
             info_dict = ydl.extract_info(url, download=False)
+            # print(json.dumps(info_dict))
             title = info_dict.get("title", "unknow")
             seconds = info_dict.get("duration")
             duration = str(seconds) if seconds else ""
@@ -72,21 +157,46 @@ async def fetchSubtitlebyType(url: str, lang: Optional[str] = 'en', subType: Opt
             if info_dict.get("extractor") == "youtube" and subType == "srt":
                 subType = "xml"
                 isSrt = True
+            if debug:
+                print(
+                    "subtitles.keys(): {} automatic_captions: {}".format(
+                        info_dict.get("subtitles").keys(),
+                        info_dict.get("automatic_captions").keys(),
+                    )
+                )
+
+            subtitle_url = getRequestedSubtitlesUrl(info_dict, lang, subType)
+            if not subtitle_url:
+                subtitle_url = getSubtitleLangUrl(info_dict, lang, subType)
+            if not subtitle_url:
+                subtitle_url = getSubtitleOtherUrl(info_dict, lang, subType)
 
-            # print("subtitles.keys(): {} automatic_captions: {}".format( info_dict.get("subtitles").keys(),info_dict.get("automatic_captions").keys()))
-            for subtitle_item in ["subtitles", "automatic_captions"]: # "requested_subtitles" item is dict
-                if info_dict.get(subtitle_item):
-                    subtitle_url = getUrlFromSubtitles(info_dict.get(subtitle_item), lang, subType)
-                    if subtitle_url:
-                        # print("subtitle_url: {}".format(subtitle_url))
-                        with ydl.urlopen(subtitle_url) as response:
-                            subtitle = xml_caption_to_srt(response.read().decode()) if isSrt else response.read().decode()
-                            print("url{}, title:{}, duration:{} len(subtitle): {}".format(url, title, duration, len(subtitle)))
-                            return {"title": title, "duration": duration,"subtitle": subtitle, "chapters":info_dict.get("chapters", None) }
+            if subtitle_url:
+                # print("subtitle_url: {}".format(subtitle_url))
+                with ydl.urlopen(subtitle_url) as response:
+                    subtitle = (
+                        xml_caption_to_srt(response.read().decode())
+                        if isSrt
+                        else response.read().decode()
+                    )
+                    print(
+                        "url:{}, title:{}, duration:{} len(subtitle): {}".format(
+                            url, title, duration, len(subtitle)
+                        )
+                    )
+                    return {
+                        "title": title,
+                        "duration": duration,
+                        "subtitle": subtitle,
+                        "chapters": info_dict.get("chapters", None),
+                    }
     except Exception as e:
+        print(e)
+        traceback.print_exc()
         return {"error": str(e)}
     return {"title": title, "duration": duration, "error": "No subtitles"}
 
+
 def float_to_srt_time_format(d: float) -> str:
     """Convert decimal durations into proper srt format.
     :rtype: str
@@ -99,7 +209,8 @@ def float_to_srt_time_format(d: float) -> str:
     ms = f"{fraction:.3f}".replace("0.", "")
     return time_fmt + ms
 
-def xml_caption_to_srt( xml_captions: str) -> str:
+
+def xml_caption_to_srt(xml_captions: str) -> str:
     """Convert xml caption tracks to "SubRip Subtitle (srt)".
     :param str xml_captions:
         XML formatted caption tracks.
@@ -108,7 +219,9 @@ def xml_caption_to_srt( xml_captions: str) -> str:
     root = ElementTree.fromstring(xml_captions)
     for i, child in enumerate(list(root)):
         text = child.text or ""
-        caption = unescape(text.replace("\n", " ").replace("  ", " "),)
+        caption = unescape(
+            text.replace("\n", " ").replace("  ", " "),
+        )
         try:
             duration = float(child.attrib["dur"])
         except KeyError:
@@ -125,25 +238,27 @@ def xml_caption_to_srt( xml_captions: str) -> str:
         segments.append(line)
     return "\n".join(segments).strip()
 
+
 async def fetchSubtitleUrls(url: str, proxy: Optional[str] = None) -> json:
     ydl_opts = {
         "noplaylist": True,
-        "writesubtitles": False,
-        "allsubtitles": True,
-        "skip_download": True,
+        # "writesubtitles": False,
+        # "allsubtitles": False,
+        "listsubtitles": True,
+        # "skip_download": True,
         "socket_timeout": 10,
         "extractor_retries": 0,
-        # "extractor_args": {
-        #     "youtube": {
-        #         "player_skip": ["webpage", "configs", "initial"],
-        #         "player_client": ["android"],
-        #         "skip": ["hls", "dash", "translated_subs"],
-        #     }
-        # },
+        "extractor_args": {
+            "youtube": {
+                "player_skip": ["configs", "initial"],  # "webpage",
+                "player_client": ["web"],
+                "skip": ["hls", "dash"],  # , "translated_subs"
+            }
+        },
     }
     if proxy:
         ydl_opts.update({"proxy": proxy, "socket_timeout": 20})
-    
+
     title = "unknow"
     duration = ""
     try:
@@ -153,7 +268,12 @@ async def fetchSubtitleUrls(url: str, proxy: Optional[str] = None) -> json:
             seconds = info_dict.get("duration")
             duration = str(seconds) if seconds else ""
 
-            return {"title": title, "duration": duration, "subtitles": info_dict.get("subtitles"),"automatic_captions": info_dict.get("automatic_captions")}
+            return {
+                "title": title,
+                "duration": duration,
+                "subtitles": info_dict.get("subtitles"),
+                "automatic_captions": info_dict.get("automatic_captions"),
+            }
 
     except Exception as e:
         return {"error": str(e)}