🍣 removed scrap and simplified the urlResolver

src/common/settings.ts

@@ -13,8 +13,8 @@ export function getExtensionSettingsAsync(): Promise<ExtensionSettings> {
 }
 
 
-export type TargetPlatformName = 'madiator.com' | 'odysee' | 'app'
-export interface TargetPlatform {
+
+const targetPlatform = (o: {
   domainPrefix: string
   displayName: string
   theme: string
@@ -27,10 +27,14 @@ export interface TargetPlatform {
       button?: JSX.CSSProperties
     }
   }
+}) => o
+export type TargetPlatform = ReturnType<typeof targetPlatform>
+export type TargetPlatformName = Extract<keyof typeof targetPlatformSettings, string>
+export const getTargetPlatfromSettingsEntiries = () => {
+  return Object.entries(targetPlatformSettings) as any as [Extract<keyof typeof targetPlatformSettings, string>, TargetPlatform][]
 }
-
-export const targetPlatformSettings: Record<TargetPlatformName, TargetPlatform> = {
-  'madiator.com': {
+export const targetPlatformSettings = {
+  'madiator.com': targetPlatform({
     domainPrefix: 'https://madiator.com/',
     displayName: 'Madiator.com',
     theme: '#075656',
@@ -42,8 +46,8 @@ export const targetPlatformSettings: Record<TargetPlatformName, TargetPlatform>
         icon: { transform: 'scale(1.2)' }
       }
     }
-  },
-  odysee: {
+  }),
+  odysee: targetPlatform({
     domainPrefix: 'https://odysee.com/',
     displayName: 'Odysee',
     theme: '#1e013b',
@@ -51,8 +55,8 @@ export const targetPlatformSettings: Record<TargetPlatformName, TargetPlatform>
       text: 'Watch on Odysee',
       icon: chrome.runtime.getURL('icons/lbry/odysee-logo.svg')
     }
-  },
-  app: {
+  }),
+  app: targetPlatform({
     domainPrefix: 'lbry://',
     displayName: 'LBRY App',
     theme: '#075656',
@@ -60,118 +64,59 @@ export const targetPlatformSettings: Record<TargetPlatformName, TargetPlatform>
       text: 'Watch on LBRY',
       icon: chrome.runtime.getURL('icons/lbry/lbry-logo.svg')
     }
-  },
-}
-
-export const getTargetPlatfromSettingsEntiries = () => {
-  return Object.entries(targetPlatformSettings) as any as [Extract<keyof typeof targetPlatformSettings, string>, TargetPlatform][]
+  }),
 }
 
 
 
-export type SourcePlatformName = 'youtube.com' | 'yewtu.be'
-export interface SourcePlatform {
+
+
+const sourcePlatform = (o: {
   hostnames: string[]
   htmlQueries: {
     mountButtonBefore: string,
     videoPlayer: string
   }
-}
-
-export const sourcePlatfromSettings: Record<SourcePlatformName, SourcePlatform> = {
-  "yewtu.be": {
-    hostnames: ['yewtu.be', 'vid.puffyan.us', 'invidio.xamh.de', 'invidious.kavin.rocks'],
-    htmlQueries: {
-      mountButtonBefore: '#watch-on-youtube',
-      videoPlayer: '#player-container video'
-    }
-  },
-  "youtube.com": {
-    hostnames: ['www.youtube.com'],
-    htmlQueries: {
-      mountButtonBefore: 'ytd-video-owner-renderer~#subscribe-button',
-      videoPlayer: '#ytd-player video'
-    }
-  }
-}
-
+}) => o
+export type SourcePlatform = ReturnType<typeof sourcePlatform>
+export type SourcePlatformName = Extract<keyof typeof sourcePlatfromSettings, string>
 export function getSourcePlatfromSettingsFromHostname(hostname: string) {
   const values = Object.values(sourcePlatfromSettings)
   for (const settings of values)
     if (settings.hostnames.includes(hostname)) return settings
   return null
 }
-
-
-export type YTUrlResolverName = 'lbryInc' | 'madiatorScrap'
-
-export const Keys = Symbol('keys')
-export const Values = Symbol('values')
-export const SingleValueAtATime = Symbol()
-export type YtUrlResolveResponsePath = (string | number | typeof Keys | typeof Values)[]
-export interface YtUrlResolveFunction {
-  pathname: string
-  defaultParams: Record<string, string | number>
-  valueParamName: string
-  paramArraySeperator: string | typeof SingleValueAtATime
-  responsePath: YtUrlResolveResponsePath
+export const sourcePlatfromSettings = {
+  "yewtu.be": sourcePlatform({
+    hostnames: ['yewtu.be', 'vid.puffyan.us', 'invidio.xamh.de', 'invidious.kavin.rocks'],
+    htmlQueries: {
+      mountButtonBefore: '#watch-on-youtube',
+      videoPlayer: '#player-container video'
+    }
+  }),
+  "youtube.com": sourcePlatform({
+    hostnames: ['www.youtube.com'],
+    htmlQueries: {
+      mountButtonBefore: 'ytd-video-owner-renderer~#subscribe-button',
+      videoPlayer: '#ytd-player video'
+    }
+  })
 }
-export interface YTUrlResolver {
+
+const ytUrlResolver = (o: {
   name: string
-  hostname: string
-  functions: {
-    getChannelId: YtUrlResolveFunction
-    getVideoId: YtUrlResolveFunction
-  }
-}
-
-export const ytUrlResolversSettings: Record<YTUrlResolverName, YTUrlResolver> = {
-  lbryInc: {
+  href: string
+}) => o
+export type YTUrlResolver = ReturnType<typeof ytUrlResolver>
+export type YTUrlResolverName = Extract<keyof typeof ytUrlResolversSettings, string>
+export const getYtUrlResolversSettingsEntiries = () => Object.entries(ytUrlResolversSettings) as any as [Extract<keyof typeof ytUrlResolversSettings, string>, YTUrlResolver][]
+export const ytUrlResolversSettings = {
+  lbryInc: ytUrlResolver({
     name: "Odysee",
-    hostname: "api.odysee.com",
-    functions: {
-      getChannelId: {
-        pathname: "/yt/resolve",
-        defaultParams: {},
-        valueParamName: "channel_ids",
-        paramArraySeperator: ',',
-        responsePath: ["data", "channels", Values]
-      },
-      getVideoId: {
-        pathname: "/yt/resolve",
-        defaultParams: {},
-        valueParamName: "video_ids",
-        paramArraySeperator: ",",
-        responsePath: ["data", "videos", Values]
-      }
-    }
-  },
-  madiatorScrap: {
-    name: "Madiator.com",
-    hostname: "scrap.madiator.com",
-    functions: {
-      getChannelId: {
-        pathname: "/api/get-lbry-channel",
-        defaultParams: {
-          v: 2
-        },
-        valueParamName: "url",
-        paramArraySeperator: SingleValueAtATime,
-        responsePath: ["lbrych"]
-      },
-      getVideoId: {
-        pathname: "/api/get-lbry-video",
-        defaultParams: {
-          v: 2
-        },
-        valueParamName: "url",
-        paramArraySeperator: SingleValueAtATime,
-        responsePath: ["lbryurl"]
-      }
-    }
-  }
-}
-
-export const getYtUrlResolversSettingsEntiries = () => {
-  return Object.entries(ytUrlResolversSettings) as any as [Extract<keyof typeof ytUrlResolversSettings, string>, YTUrlResolver][]
+    href: "https://api.odysee.com/yt/resolve"
+  }),
+  madiatorFinder: ytUrlResolver({
+    name: "Madiator Finder",
+    href: "https://finder.madiator.com/api/v1/resolve"
+  })
 }

src/common/yt/auth.ts

@@ -0,0 +1,32 @@
+import crypto from 'crypto'
+
+function generateKeys() {
+    // The `generateKeyPairSync` method accepts two arguments:
+    // 1. The type ok keys we want, which in this case is "rsa"
+    // 2. An object with the properties of the key
+    const keys = crypto.generateKeyPairSync("rsa", {
+        // The standard secure default length for RSA keys is 2048 bits
+        modulusLength: 2048,
+    })
+
+    return keys
+}
+
+function encodeKeys(keys: { publicKey: Buffer, privateKey: Buffer }) {
+    return JSON.stringify({ publicKey: keys.publicKey.toString('base64'), privateKey: keys.privateKey.toString('base64') })
+}
+
+function decodeKeys(encodedKeys: string) {
+    const keysBase64 = JSON.parse(encodedKeys)
+    return {
+        publicKey: Buffer.from(keysBase64.publicKey),
+        privateKey: Buffer.from(keysBase64.privateKey)
+    }
+}
+
+function sign(data: string, privateKey: Buffer) {
+    return crypto.sign("sha256", Buffer.from(data), {
+        key: privateKey,
+        padding: crypto.constants.RSA_PKCS1_PSS_PADDING,
+    })
+}

src/common/yt/urlResolve.ts

@@ -1,105 +1,80 @@
-import { chunk, groupBy } from "lodash"
-import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YtUrlResolveResponsePath, ytUrlResolversSettings } from "../settings"
+import { chunk } from "lodash"
+import { getExtensionSettingsAsync, ytUrlResolversSettings } from "../settings"
 import { LbryPathnameCache } from "./urlCache"
 
 // const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS
 const QUERY_CHUNK_SIZE = 100
 
-export interface YtIdResolverDescriptor {
-    id: string
-    type: 'channel' | 'video'
+
+export type YtUrlResolveItem = { type: 'video' | 'channel', id: string }
+type Results = Record<string, YtUrlResolveItem>
+type Paramaters = YtUrlResolveItem[]
+
+interface ApiResponse {
+    channels?: Record<string, string>
+    videos?: Record<string, string>
 }
 
-/**
-* @param descriptorsWithIndex YT resource IDs to check
-* @returns a promise with the list of channels that were found on lbry
-*/
-export async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]> {
-    let descriptorsPayload: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({ ...descriptor, index }))
-    descriptors = null as any
-    const results: (string | null)[] = []
+export async function resolveById(params: Paramaters, progressCallback?: (progress: number) => void): Promise<Results> {
+    const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync()
+    const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName]
 
+    async function requestChunk(params: Paramaters) {
+        const results: Results = {}
 
-    descriptorsPayload = (await Promise.all(descriptorsPayload.map(async (descriptor, index) => {
-        if (!descriptor?.id) return
-        const cache = await LbryPathnameCache.get(descriptor.id)
+        // Check for cache first, add them to the results if there are any cache
+        // And remove them from the params, so we dont request for them
+        params = (await Promise.all(params.map(async (item) => {
+            const cachedLbryUrl = await LbryPathnameCache.get(item.id)
 
-        // Cache can be null, if there is no lbry url yet
-        if (cache !== undefined) {
-            // Null values shouldn't be in the results
-            if (cache) results[index] = cache
-            return
-        }
-
-        return descriptor
-    }))).filter((descriptor) => descriptor) as any
-
-    const descriptorsPayloadChunks = chunk(descriptorsPayload, QUERY_CHUNK_SIZE)
-    let progressCount = 0
-    await Promise.all(descriptorsPayloadChunks.map(async (descriptorChunk) => {
-        const descriptorsGroupedByType: Record<YtIdResolverDescriptor['type'], typeof descriptorsPayload | null> = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any
-
-        const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync()
-        const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName]
-
-        const url = new URL(`https://${urlResolverSetting.hostname}`)
-
-        function followResponsePath<T>(response: any, responsePath: YtUrlResolveResponsePath) {
-            for (const path of responsePath) {
-                switch (typeof path) {
-                    case 'string': case 'number': response = response[path]; continue
-                }
-                switch (path) {
-                    case Keys: response = Object.keys(response); continue
-                    case Values: response = Object.values(response); continue
-                }
+            // Cache can be null, if there is no lbry url yet
+            if (cachedLbryUrl !== undefined) {
+                // Null values shouldn't be in the results
+                if (cachedLbryUrl !== null) results[item.id] = { id: cachedLbryUrl, type: item.type }
+                return null
             }
-            return response as T
-        }
 
-        async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsPayload) {
-            url.pathname = urlResolverFunction.pathname
-            Object.entries(urlResolverFunction.defaultParams).forEach(([name, value]) => url.searchParams.set(name, value.toString()))
+            // No cache found
+            return item
+        }))).filter((o) => o) as Paramaters
 
-            if (urlResolverFunction.paramArraySeperator === SingleValueAtATime) {
-                await Promise.all(descriptorsGroup.map(async (descriptor) => {
-                    url.searchParams.set(urlResolverFunction.valueParamName, descriptor.id)
+        if (params.length === 0) return results
 
-                    const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
-                    if (apiResponse.ok) {
-                        const value = followResponsePath<string>(await apiResponse.json(), urlResolverFunction.responsePath)
-                        if (value) results[descriptor.index] = value
-                        await LbryPathnameCache.put(value, descriptor.id)
-                    }
-                    else if (apiResponse.status === 404) await LbryPathnameCache.put(null, descriptor.id)
+        const url = new URL(`${urlResolverSetting.href}`)
+        url.searchParams.set('video_ids', params.filter((item) => item.type === 'video').map((item) => item.id).join(','))
+        url.searchParams.set('channel_ids', params.filter((item) => item.type === 'channel').map((item) => item.id).join(','))
 
-                    progressCount++
-                    if (progressCallback) progressCallback(progressCount / descriptorsPayload.length)
-                }))
+        const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
+        if (apiResponse.ok) {
+            const response: ApiResponse = await apiResponse.json()
+            for (const [id, lbryUrl] of Object.entries(response.channels ?? {})) {
+                // we cache it no matter if its null or not
+                await LbryPathnameCache.put(lbryUrl, id)
+
+                if (lbryUrl) results[id] = { id: lbryUrl, type: 'channel' }
             }
-            else {
-                url.searchParams.set(urlResolverFunction.valueParamName, descriptorsGroup
-                    .map((descriptor) => descriptor.id)
-                    .join(urlResolverFunction.paramArraySeperator))
+            for (const [id, lbryUrl] of Object.entries(response.videos ?? {})) {
+                // we cache it no matter if its null or not
+                await LbryPathnameCache.put(lbryUrl, id)
 
-                const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
-                if (apiResponse.ok) {
-                    const values = followResponsePath<string[]>(await apiResponse.json(), urlResolverFunction.responsePath)
-                    await Promise.all(values.map(async (value, index) => {
-                        const descriptor = descriptorsGroup[index]
-                        if (value) results[descriptor.index] = value
-                        await LbryPathnameCache.put(value, descriptor.id)
-                    }))
-                }
-
-                progressCount += descriptorsGroup.length
-                if (progressCallback) progressCallback(progressCount / descriptorsPayload.length)
+                if (lbryUrl) results[id] = { id: lbryUrl, type: 'video' }
             }
         }
 
-        if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel'])
-        if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video'])
-    }))
+        return results
+    }
+
+    const results: Results = {}
+    const chunks = chunk(params, QUERY_CHUNK_SIZE)
+
+    let i = 0
+    if (progressCallback) progressCallback(0)
+    for (const chunk of chunks) {
+        if (progressCallback) progressCallback(++i / (chunks.length + 1))
+        Object.assign(results, await requestChunk(chunk))
+    }
+    console.log(results)
+
     if (progressCallback) progressCallback(1)
     return results
 }

src/manifest.json

@@ -11,7 +11,7 @@
     "https://lbry.tv/",
     "https://odysee.com/",
     "https://madiator.com/",
-    "https://scrap.madiator.com/",
+    "https://finder.madiator.com/",
     "tabs",
     "storage"
   ],

src/scripts/background.ts

@@ -1,7 +1,7 @@
-import { resolveById, YtIdResolverDescriptor } from '../common/yt/urlResolve'
+import { resolveById, YtUrlResolveItem } from '../common/yt/urlResolve'
 
-async function resolveYT(descriptor: YtIdResolverDescriptor) {
-  const lbryProtocolUrl: string | null = (await resolveById([descriptor]).then(a => a[0])) ?? null
+async function resolveYT(item: YtUrlResolveItem) {
+  const lbryProtocolUrl: string | null = (await resolveById([item]).then((items) => items[item.id]))?.id ?? null
   if (!lbryProtocolUrl) return null
   return lbryProtocolUrl.replaceAll('#', ':')
   /* const segments = parseProtocolUrl(lbryProtocolUrl || '', { encode: true })

src/tools/YTtoLBRY.tsx

@@ -18,12 +18,14 @@ async function lbryChannelsFromFile(file: File) {
     ext === 'xml' || ext == 'opml' ? getSubsFromOpml :
       ext === 'csv' ? getSubsFromCsv :
         getSubsFromJson)(await getFileContent(file)))
-  const lbryPathnames = await resolveById(
+
+  const items = await resolveById(
     Array.from(ids).map(id => ({ id, type: 'channel' } as const)),
     (progress) => render(<YTtoLBRY progress={progress} />, document.getElementById('root')!))
+
   const { targetPlatform: platform } = await getExtensionSettingsAsync()
   const urlPrefix = targetPlatformSettings[platform].domainPrefix
-  return lbryPathnames.map(channel => urlPrefix + channel)
+  return Object.values(items).map((item) => urlPrefix + item.id)
 }
 
 function ConversionCard({ onSelect, progress }: { onSelect(file: File): Promise<void> | void, progress: number }) {