faster download of public channels

src/app/server/actions/ai-tube-hf/getChannel.ts

@@ -7,8 +7,8 @@ import { getChannels } from "./getChannels"
 
 export async function getChannel(options: {
   channelId?: string | string[] | null
-  apiKey?: string
-  owner?: string
+  // apiKey?: string // deprecated, we only work on public content
+  // owner?: string // deprecated, we only work on public content
   renewCache?: boolean
   neverThrow?: boolean
 } = {}): Promise<ChannelInfo | undefined> {
@@ -20,8 +20,8 @@ export async function getChannel(options: {
 
     const channels = await getChannels({
       channelId: id,
-      apiKey: options?.apiKey,
-      owner: options?.owner,
+      // apiKey: options?.apiKey, // deprecated, we only work on public content
+      // owner: options?.owner, // deprecated, we only work on public content
       renewCache: options.renewCache,
     })
 

src/app/server/actions/ai-tube-hf/getChannelVideos.ts

@@ -4,7 +4,7 @@ import { ChannelInfo, VideoInfo, VideoStatus } from "@/types"
 
 import { getVideoRequestsFromChannel  } from "./getVideoRequestsFromChannel"
 import { adminApiKey } from "../config"
-import { getIndex } from "./getIndex"
+import { getVideoIndex } from "./getVideoIndex"
 
 // return 
 export async function getChannelVideos({
@@ -25,8 +25,8 @@ export async function getChannelVideos({
 
   // TODO: use a database instead
   // normally 
-  const queued = await getIndex({ status: "queued" })
-  const published = await getIndex({ status: "published" })
+  const queued = await getVideoIndex({ status: "queued" })
+  const published = await getVideoIndex({ status: "published" })
 
   return videos.map(v => {
    let video: VideoInfo = {

src/app/server/actions/ai-tube-hf/getChannels.ts

@@ -1,87 +1,47 @@
 "use server"
 
-import { Credentials, listDatasets, whoAmI } from "@/huggingface/hub/src"
 import { ChannelInfo } from "@/types"
 
-import { adminCredentials } from "../config"
-import { parseChannel } from "./parseChannel"
+import { adminUsername } from "../config"
 
-export async function getChannels(options: {
+export async function getChannels({
+  channelId = "",
+  renewCache = true,
+  neverThrow = true
+}: {
   channelId?: string
-  apiKey?: string
-  owner?: string
   renewCache?: boolean
+  neverThrow?: boolean
 } = {}): Promise<ChannelInfo[]> {
-  // console.log("getChannels")
-  let credentials: Credentials = adminCredentials
-  let owner = options?.owner
-
-  if (options?.apiKey) {
-    try {
-      credentials = { accessToken: options.apiKey }
-      const { name: username } = await whoAmI({ credentials })
-      if (!username) {
-        throw new Error(`couldn't get the username`)
-      }
-      // everything is in order,
-      owner = username
-    } catch (err) {
-      console.error(err)
-      return []
-    }
-  }
-
-  let channels: ChannelInfo[] = []
-  
-  const prefix = "ai-tube-"
-
-  let search = owner
-    ? { owner } // search channels of a specific user
-    : prefix // global search (note: might be costly?)
-
-  // console.log("search:", search)
-  
-  for await (const { id, name, likes, updatedAt } of listDatasets({
-    search,
-    credentials,
-    requestInit: options?.renewCache
-    ? { cache: "no-store" }
-    : undefined
-  })) {
-    if (options.channelId && options.channelId !== id) {
-      continue
-    }
-
-    // TODO: need to handle better cases where the username is missing
+  try {
+    const response = await fetch(
+      `https://huggingface.co/datasets/${adminUsername}/ai-tube-index/raw/main/channels.json`
+    , {
+      cache: renewCache ? "no-store" : "default"
+    })
 
-    const chunks = name.split("/")
-    const [_datasetUser, datasetName] = chunks.length === 2
-      ? chunks
-      : [name, name]
+    const jsonResponse = await response?.json()
 
-    // console.log(`found a candidate dataset "${datasetName}" owned by @${datasetUser}`)
-    
-    // ignore channels which don't start with ai-tube
-    if (!datasetName.startsWith(prefix)) {
-      continue
+    if (
+      typeof jsonResponse === "undefined" ||
+      typeof jsonResponse !== "object" ||
+      Array.isArray(jsonResponse) ||
+      jsonResponse === null) {
+      throw new Error("index is not an object, admin repair needed")
     }
 
-    // ignore the video index
-    if (datasetName === "ai-tube-index") {
-      continue
-    }
+    const channelsIndex = (jsonResponse as Record<string, ChannelInfo>) || {}
 
-    const channel = await parseChannel({
-      ...options,
-      id, name, likes, updatedAt
-    })
+    const channels = Object.values(channelsIndex)
 
-    channels.push(channel)
+    if (!channelId) { return channels }
 
-    if (options.channelId && options.channelId === id) {
-      break
+    return channels.filter(channel => channel.id === channelId)
+  } catch (err) {
+    if (neverThrow) {
+      console.error(`failed to get the channel index:`, err)
+      return []
     }
+    throw err
   }
-
-  return channels
 }

src/app/server/actions/ai-tube-hf/getPrivateChannels.ts

@@ -0,0 +1,87 @@
+"use server"
+
+import { Credentials, listDatasets, whoAmI } from "@/huggingface/hub/src"
+import { ChannelInfo } from "@/types"
+
+import { adminCredentials } from "../config"
+import { parseChannel } from "./parseChannel"
+
+export async function getPrivateChannels(options: {
+  channelId?: string
+  apiKey?: string
+  owner?: string
+  renewCache?: boolean
+} = {}): Promise<ChannelInfo[]> {
+  // console.log("getChannels")
+  let credentials: Credentials = adminCredentials
+  let owner = options?.owner
+
+  if (options?.apiKey) {
+    try {
+      credentials = { accessToken: options.apiKey }
+      const { name: username } = await whoAmI({ credentials })
+      if (!username) {
+        throw new Error(`couldn't get the username`)
+      }
+      // everything is in order,
+      owner = username
+    } catch (err) {
+      console.error(err)
+      return []
+    }
+  }
+
+  let channels: ChannelInfo[] = []
+  
+  const prefix = "ai-tube-"
+
+  let search = owner
+    ? { owner } // search channels of a specific user
+    : prefix // global search (note: might be costly?)
+
+  // console.log("search:", search)
+  
+  for await (const { id, name, likes, updatedAt } of listDatasets({
+    search,
+    credentials,
+    requestInit: options?.renewCache
+    ? { cache: "no-store" }
+    : undefined
+  })) {
+    if (options.channelId && options.channelId !== id) {
+      continue
+    }
+
+    // TODO: need to handle better cases where the username is missing
+
+    const chunks = name.split("/")
+    const [_datasetUser, datasetName] = chunks.length === 2
+      ? chunks
+      : [name, name]
+
+    // console.log(`found a candidate dataset "${datasetName}" owned by @${datasetUser}`)
+    
+    // ignore channels which don't start with ai-tube
+    if (!datasetName.startsWith(prefix)) {
+      continue
+    }
+
+    // ignore the video index
+    if (datasetName === "ai-tube-index") {
+      continue
+    }
+
+    const channel = await parseChannel({
+      ...options,
+      id, name, likes, updatedAt
+    })
+
+    channels.push(channel)
+
+    if (options.channelId && options.channelId === id) {
+      break
+    }
+  }
+
+  return channels
+}

src/app/server/actions/ai-tube-hf/getTags.ts

@@ -1,6 +1,6 @@
 "use server"
 
-import { getIndex } from "./getIndex"
+import { getVideoIndex } from "./getVideoIndex"
 
 export async function getTags({
   renewCache = true,
@@ -13,7 +13,7 @@ export async function getTags({
   neverThrow: true,
 }): Promise<string[]> {
   try {
-    const published = Object.values(await getIndex({
+    const published = Object.values(await getVideoIndex({
       status: "published",
       renewCache,
     }))

src/app/server/actions/ai-tube-hf/getVideo.ts

@@ -2,7 +2,7 @@
 
 import { VideoInfo } from "@/types"
 
-import { getIndex } from "./getIndex"
+import { getVideoIndex } from "./getVideoIndex"
 
 export async function getVideo({
   videoId,
@@ -17,7 +17,7 @@ export async function getVideo({
     if (!id) {
       throw new Error(`cannot get the video, invalid id: "${id}"`)
     }
-    const published = await getIndex({ status: "published" })
+    const published = await getVideoIndex({ status: "published" })
 
     const video = published[id] || undefined
 

src/app/server/actions/ai-tube-hf/{getIndex.ts → getVideoIndex.ts}

@@ -2,7 +2,7 @@ import { VideoInfo, VideoStatus } from "@/types"
 
 import { adminUsername } from "../config"
 
-export async function getIndex({
+export async function getVideoIndex({
   status,
   renewCache = true,
   neverThrow = true,
@@ -22,8 +22,8 @@ export async function getIndex({
     const jsonResponse = await response?.json()
 
     if (
-      typeof jsonResponse === "undefined" &&
-      typeof jsonResponse !== "object" &&
+      typeof jsonResponse === "undefined" ||
+      typeof jsonResponse !== "object" ||
       Array.isArray(jsonResponse) ||
       jsonResponse === null) {
       throw new Error("index is not an object, admin repair needed")

src/app/server/actions/ai-tube-hf/getVideos.ts

@@ -2,7 +2,7 @@
 
 import { VideoInfo } from "@/types"
 
-import { getIndex } from "./getIndex"
+import { getVideoIndex } from "./getVideoIndex"
 
 const HARD_LIMIT = 100
 
@@ -31,7 +31,7 @@ export async function getVideos({
 }): Promise<VideoInfo[]> {
   // the index is gonna grow more and more,
   // but in the future we will use some DB eg. Prisma or sqlite
-  const published = await getIndex({
+  const published = await getVideoIndex({
     status: "published",
     renewCache: true
   })

src/app/views/user-account-view/index.tsx

@@ -5,12 +5,12 @@ import { useLocalStorage } from "usehooks-ts"
 
 import { useStore } from "@/app/state/useStore"
 import { cn } from "@/lib/utils"
-import { getChannels } from "@/app/server/actions/ai-tube-hf/getChannels"
 import { ChannelList } from "@/app/interface/channel-list"
 import { localStorageKeys } from "@/app/state/localStorageKeys"
 import { defaultSettings } from "@/app/state/defaultSettings"
 import { Input } from "@/components/ui/input"
-import { ChannelInfo } from "@/types"
+
+import { getPrivateChannels } from "@/app/server/actions/ai-tube-hf/getPrivateChannels"
 
 export function UserAccountView() {
   const [_isPending, startTransition] = useTransition()
@@ -29,7 +29,7 @@ export function UserAccountView() {
     if (!isLoaded) {
       startTransition(async () => {
         try {
-          const channels = await getChannels({
+          const channels = await getPrivateChannels({
             apiKey: huggingfaceApiKey,
             renewCache: true,
           })