Merge pull request #77 from DeepDoge/patch

[Feature] Madiator.com URL resolver added

src/common/settings.ts

@@ -1,9 +1,10 @@
 export interface ExtensionSettings {
   redirect: boolean
   targetPlatform: TargetPlatformName
+  urlResolver: YTUrlResolverName
 }
 
-export const DEFAULT_SETTINGS: ExtensionSettings = { redirect: true, targetPlatform: 'odysee' };
+export const DEFAULT_SETTINGS: ExtensionSettings = { redirect: true, targetPlatform: 'odysee', urlResolver: 'lbryInc' };
 
 export function getExtensionSettingsAsync<K extends Array<keyof ExtensionSettings>>(...keys: K): Promise<Pick<ExtensionSettings, K[number]>> {
   return new Promise(resolve => chrome.storage.local.get(keys, o => resolve(o as any)));
@@ -73,4 +74,71 @@ export function getSourcePlatfromSettingsFromHostname(hostname: string) {
   for (const settings of values)
     if (settings.hostnames.includes(hostname)) return settings
   return null
+}
+
+
+export type YTUrlResolverName = 'lbryInc' | 'madiatorScrap' 
+
+export const Keys = Symbol('keys')
+export const Values = Symbol('values')
+export const SingleValueAtATime = Symbol()
+export type YtUrlResolveResponsePath = (string | number | typeof Keys | typeof Values)[]
+export interface YtUrlResolveFunction
+{
+  pathname: string
+  paramName: string
+  paramArraySeperator: string | typeof SingleValueAtATime
+  responsePath: YtUrlResolveResponsePath
+}
+export interface YTUrlResolver
+{
+  name: string
+  hostname: string
+  functions: {
+    getChannelId: YtUrlResolveFunction
+    getVideoId: YtUrlResolveFunction
+  }
+}
+
+export const ytUrlResolversSettings: Record<YTUrlResolverName, YTUrlResolver> = {
+  lbryInc: {
+    name: "LBRY Inc.",
+    hostname: "api.odysee.com",
+    functions: {
+      getChannelId : {
+        pathname: "/yt/resolve",
+        paramName: "channel_ids",
+        paramArraySeperator: ',',
+        responsePath: ["data", "channels", Values]
+      },
+      getVideoId: {
+        pathname: "/yt/resolve",
+        paramName: "video_ids",
+        paramArraySeperator: ",",
+        responsePath: ["data", "videos", Values]
+      }
+    }
+  },
+  madiatorScrap: {
+    name: "Madiator.com",
+    hostname: "scrap.madiator.com",
+    functions: {
+      getChannelId: {
+        pathname: "/api/get-lbry-channel",
+        paramName: "url",
+        paramArraySeperator: SingleValueAtATime,
+        responsePath: ["lbrych"]
+      },
+      getVideoId: {
+        pathname: "/api/get-lbry-video",
+        paramName: "url",
+        paramArraySeperator: SingleValueAtATime,
+        responsePath: ["lbryurl"]
+      }
+    }
+  }
+}
+
+export const getYtUrlResolversSettingsEntiries = () => {
+  return Object.entries(ytUrlResolversSettings) as any as [Extract<keyof typeof ytUrlResolversSettings, string>, YTUrlResolver][]
 }

src/common/yt.ts

@@ -1,19 +1,10 @@
 import chunk from 'lodash/chunk';
 import groupBy from 'lodash/groupBy';
-import pickBy from 'lodash/pickBy';
+import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YTUrlResolver, YtUrlResolveResponsePath, ytUrlResolversSettings } from './settings'
 
-const LBRY_API_HOST = 'https://api.odysee.com';
+// const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS
 const QUERY_CHUNK_SIZE = 300;
 
-interface YtResolverResponse {
-  success: boolean;
-  error: object | null;
-  data: {
-    videos?: Record<string, string>;
-    channels?: Record<string, string>;
-  };
-}
-
 interface YtExportedJsonSubscription {
   id: string;
   etag: string;
@@ -42,7 +33,7 @@ export function getFileContent(file: File): Promise<string> {
   });
 }
 
-export interface YTDescriptor {
+export interface YtIdResolverDescriptor {
   id: string
   type: 'channel' | 'video'
 }
@@ -86,7 +77,7 @@ export const ytService = {
   readCsv(csvContent: string): string[] {
     const rows = csvContent.split('\n')
     csvContent = ''
-    return rows.map((row) => row.substring(0, row.indexOf(',')))
+    return rows.slice(1).map((row) => row.substring(0, row.indexOf(',')))
   },
 
   /**
@@ -108,7 +99,7 @@ export const ytService = {
     return match ? match[1] : null; // match[1] is the videoId
   },
 
-  getId(url: string): YTDescriptor | null {
+  getId(url: string): YtIdResolverDescriptor | null {
     const videoId = ytService.getVideoId(url);
     if (videoId) return { id: videoId, type: 'video' };
     const channelId = ytService.getChannelId(url);
@@ -117,23 +108,93 @@ export const ytService = {
   },
 
   /**
-  * @param descriptors YT resource IDs to check
+  * @param descriptorsWithIndex YT resource IDs to check
   * @returns a promise with the list of channels that were found on lbry
   */
-  async resolveById(...descriptors: YTDescriptor[]): Promise<string[]> {
+  async resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<string[]> {
-    const descChunks = chunk(descriptors, QUERY_CHUNK_SIZE);
+    const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({...descriptor, index}))
-    const responses: (YtResolverResponse | null)[] = await Promise.all(descChunks.map(descChunk => {
+    
-      const groups = groupBy(descChunk, d => d.type);
+    const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE);
-      const params = new URLSearchParams(pickBy({
+    const results: string[] = []
-        video_ids: groups['video']?.map(s => s.id).join(','),
+    let progressCount = 0;
-        channel_ids: groups['channel']?.map(s => s.id).join(','),
+    await Promise.all(descriptorsChunks.map(async (descriptorChunk) => 
-      }));
+    {
-      return fetch(`${LBRY_API_HOST}/yt/resolve?${params}`, { cache: 'force-cache' })
+      const descriptorsGroupedByType: Record<YtIdResolverDescriptor['type'], typeof descriptorsWithIndex | null> = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any;
-        .then(rsp => rsp.ok ? rsp.json() : null);
-    }));
 
-    return responses.filter((rsp): rsp is YtResolverResponse => !!rsp)
+      const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync('urlResolver')
-      .flatMap(rsp => [...Object.values(rsp.data.videos || {}), ...Object.values(rsp.data.channels || {})])  // flatten the results into a 1D array
+      const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName]
-      .filter(s => s);
+
-  },
+      const url = new URL(`https://${urlResolverSetting.hostname}`);
-};
+
+      function followResponsePath<T>(response: any, responsePath: YtUrlResolveResponsePath) 
+      {
+        for (const path of responsePath) 
+        {
+          switch (typeof path) 
+          {
+            case 'string': 
+            case 'number':
+              response = response[path]
+              break
+            default:
+              switch (path) 
+              {
+                case Keys:
+                  response = Object.keys(response)
+                  break
+                case Values:
+                  response = Object.values(response)
+                  break
+              }
+          }
+        }
+        return response as T
+      }
+
+      async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsWithIndex)
+      {
+        url.pathname = urlResolverFunction.pathname
+        if (urlResolverFunction.paramArraySeperator === SingleValueAtATime)
+        {
+          for (const descriptor of descriptorsGroup)
+          {
+            progressCount++
+            if (!descriptor.id) continue
+            url.searchParams.set(urlResolverFunction.paramName, descriptor.id)
+
+            const apiResponse = await fetch(url.toString(), { cache: 'force-cache' });
+            if (!apiResponse.ok) continue
+            const value = followResponsePath<string>(await apiResponse.json(), urlResolverFunction.responsePath)
+            if (value) results[descriptor.index] = value
+            if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
+          }
+        }
+        else
+        {
+          progressCount += descriptorsGroup.length
+          url.searchParams
+            .set(urlResolverFunction.paramName, descriptorsGroup
+              .map((descriptor) => descriptor.id)
+              .filter((descriptorId) => descriptorId)
+              .join(urlResolverFunction.paramArraySeperator)
+            )
+
+          const apiResponse = await fetch(url.toString(), { cache: 'force-cache' });
+          if (!apiResponse.ok) return
+          const values = followResponsePath<string[]>(await apiResponse.json(), urlResolverFunction.responsePath)
+          values.forEach((value, index) => {
+            const descriptorIndex = descriptorsGroup[index].index
+            if (value) (results[descriptorIndex] = value)
+            if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
+          })
+        }
+      }
+
+      if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel'])
+      if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video'])
+
+    }));
+    
+    return results
+  } 
+}

src/popup/popup.tsx

@@ -1,6 +1,6 @@
 import { h, render } from 'preact'
 import ButtonRadio, { SelectionOption } from '../common/components/ButtonRadio'
-import { getTargetPlatfromSettingsEntiries, ExtensionSettings, TargetPlatformName } from '../common/settings'
+import { getTargetPlatfromSettingsEntiries, ExtensionSettings, TargetPlatformName, getYtUrlResolversSettingsEntiries, YTUrlResolverName } from '../common/settings'
 import { useLbrySettings } from '../common/useSettings'
 import './popup.sass'
 
@@ -11,8 +11,11 @@ const setSetting = <K extends keyof ExtensionSettings>(setting: K, value: Extens
 const platformOptions: SelectionOption[] = getTargetPlatfromSettingsEntiries()
   .map(([value, { displayName: display }]) => ({ value, display }));
 
+  const ytUrlResolverOptions: SelectionOption[] = getYtUrlResolversSettingsEntiries()
+  .map(([value, { name: display }]) => ({ value, display }));
+
 function WatchOnLbryPopup() {
-  const { redirect, targetPlatform } = useLbrySettings();
+  const { redirect, targetPlatform, urlResolver } = useLbrySettings();
 
   return <div className='container'>
     <section>
@@ -25,6 +28,11 @@ function WatchOnLbryPopup() {
       <ButtonRadio value={targetPlatform} options={platformOptions}
         onChange={(platform: TargetPlatformName) => setSetting('targetPlatform', platform)} />
     </section>
+    <section>
+      <label className='radio-label'>Resolve URL with:</label>
+      <ButtonRadio value={urlResolver} options={ytUrlResolverOptions}
+        onChange={(urlResolver: YTUrlResolverName) => setSetting('urlResolver', urlResolver)} />
+    </section>
     <section>
       <label className='radio-label'>Other useful tools:</label>
       <a href='/tools/YTtoLBRY.html' target='_blank'>

src/scripts/tabOnUpdated.ts

@@ -1,16 +1,16 @@
 import { appRedirectUrl, parseProtocolUrl } from '../common/lbry-url'
 import { getExtensionSettingsAsync, getSourcePlatfromSettingsFromHostname, TargetPlatformName } from '../common/settings'
-import { YTDescriptor, ytService } from '../common/yt'
+import { YtIdResolverDescriptor, ytService } from '../common/yt'
 export interface UpdateContext {
-  descriptor: YTDescriptor
+  descriptor: YtIdResolverDescriptor
   /** LBRY URL fragment */
   lbryPathname: string
   redirect: boolean
   targetPlatform: TargetPlatformName
 }
 
-async function resolveYT(descriptor: YTDescriptor) {
+async function resolveYT(descriptor: YtIdResolverDescriptor) {
-  const lbryProtocolUrl: string | null = await ytService.resolveById(descriptor).then(a => a[0]);
+  const lbryProtocolUrl: string | null = await ytService.resolveById([descriptor]).then(a => a[0]);
   const segments = parseProtocolUrl(lbryProtocolUrl || '', { encode: true });
   if (segments.length === 0) return;
   return segments.join('/');

src/tools/YTtoLBRY.css

@@ -68,6 +68,11 @@ h1, h2, h3, h4, h5, h6 {
     min-height: 30px;
 }
 
+.progress-text {
+    font-weight: bold;
+    padding: .25em 0;
+}
+
 .btn.btn-primary {
     background-color: var(--color-primary);
 }

src/tools/YTtoLBRY.tsx

@@ -19,13 +19,15 @@ async function lbryChannelsFromFile(file: File) {
     ext === 'xml' || ext == 'opml' ? ytService.readOpml : 
     ext === 'csv' ? ytService.readCsv : 
     ytService.readJson)(await getFileContent(file)))
-  const lbryUrls = await ytService.resolveById(...Array.from(ids).map(id => ({ id, type: 'channel' } as const)));
+  const lbryUrls = await ytService.resolveById(
+    Array.from(ids).map(id => ({ id, type: 'channel' } as const)), 
+    (progress) => render(<YTtoLBRY progress={progress} />, document.getElementById('root')!));
   const { targetPlatform: platform } = await getExtensionSettingsAsync('targetPlatform');
   const urlPrefix = targetPlatformSettings[platform].domainPrefix;
   return lbryUrls.map(channel => urlPrefix + channel);
 }
 
-function ConversionCard({ onSelect }: { onSelect(file: File): Promise<void> | void }) {
+function ConversionCard({ onSelect, progress }: { onSelect(file: File): Promise<void> | void, progress: number }) {
   const [file, setFile] = useState(null as File | null);
   const [isLoading, setLoading] = useState(false);
 
@@ -40,15 +42,18 @@ function ConversionCard({ onSelect }: { onSelect(file: File): Promise<void> | vo
       await onSelect(file);
       setLoading(false);
     }} />
+    <div class="progress-text">
+      {progress > 0 ? `${(progress * 100).toFixed(1)}%` : ''}
+    </div>
   </div>
 }
 
-function YTtoLBRY() {
+function YTtoLBRY({ progress }: { progress: number }) {
   const [lbryChannels, setLbryChannels] = useState([] as string[]);
 
   return <div className='YTtoLBRY'>
     <div className='Conversion'>
-      <ConversionCard onSelect={async file => setLbryChannels(await lbryChannelsFromFile(file))} />
+      <ConversionCard progress={progress} onSelect={async file => setLbryChannels(await lbryChannelsFromFile(file))} />
       <ul>
         {lbryChannels.map((x, i) => <li key={i} children={<a href={x} children={x} />} />)}
       </ul>
@@ -61,4 +66,4 @@ function YTtoLBRY() {
   </div>
 }
 
-render(<YTtoLBRY />, document.getElementById('root')!);
+render(<YTtoLBRY progress={0} />, document.getElementById('root')!);