diff --git a/src/common/yt/index.ts b/src/common/yt/index.ts index de868ee..d7ca1da 100644 --- a/src/common/yt/index.ts +++ b/src/common/yt/index.ts @@ -1,271 +1,230 @@ import chunk from 'lodash/chunk' import groupBy from 'lodash/groupBy' import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YtUrlResolveResponsePath, ytUrlResolversSettings } from '../settings' +import { LbryURLCache } from './urlCache' // const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS -const QUERY_CHUNK_SIZE = 300; +const QUERY_CHUNK_SIZE = 300 -interface YtExportedJsonSubscription { - id: string; - etag: string; - title: string; - snippet: { - description: string; - resourceId: { - channelId: string; - }; - }; +interface YtExportedJsonSubscription +{ + id: string + etag: string + title: string + snippet: { + description: string + resourceId: { + channelId: string + } + } } -export interface YtIdResolverDescriptor { - id: string - type: 'channel' | 'video' +export interface YtIdResolverDescriptor +{ + id: string + type: 'channel' | 'video' } /** * @param file to load * @returns a promise with the file as a string */ -export function getFileContent(file: File): Promise { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.addEventListener('load', event => resolve(event.target?.result as string || '')); - reader.addEventListener('error', () => { - reader.abort(); - reject(new DOMException(`Could not read ${file.name}`)); - }); - reader.readAsText(file); - }); +export function getFileContent(file: File): Promise +{ + return new Promise((resolve, reject) => + { + const reader = new FileReader() + reader.addEventListener('load', event => resolve(event.target?.result as string || '')) + reader.addEventListener('error', () => + { + reader.abort() + reject(new DOMException(`Could not read ${file.name}`)) + }) + reader.readAsText(file) + }) } -export const ytService = (() => { - /** - * Extracts the channelID from a YT URL. - * - * Handles these two types of YT URLs: - * * /feeds/videos.xml?channel_id=* - * * /channel/* - */ - function getChannelId(channelURL: string) { - const match = channelURL.match(/channel\/([^\s?]*)/); - return match ? match[1] : new URL(channelURL).searchParams.get('channel_id'); - } +/** + * Extracts the channelID from a YT URL. + * + * Handles these two types of YT URLs: + * * /feeds/videos.xml?channel_id=* + * * /channel/* + */ +export function getChannelId(channelURL: string) +{ + const match = channelURL.match(/channel\/([^\s?]*)/) + return match ? match[1] : new URL(channelURL).searchParams.get('channel_id') +} - /** - * Reads the array of YT channels from an OPML file - * - * @param opmlContents an opml file as as tring - * @returns the channel IDs - */ - function readOpml(opmlContents: string): string[] { - const opml = new DOMParser().parseFromString(opmlContents, 'application/xml'); +/** + * Reads the array of YT channels from an OPML file + * + * @param opmlContents an opml file as as tring + * @returns the channel IDs + */ +export function getSubsFromOpml(opmlContents: string): string[] +{ + const opml = new DOMParser().parseFromString(opmlContents, 'application/xml') opmlContents = '' return Array.from(opml.querySelectorAll('outline > outline')) - .map(outline => outline.getAttribute('xmlUrl')) - .filter((url): url is string => !!url) - .map(url => getChannelId(url)) - .filter((url): url is string => !!url); // we don't want it if it's empty - } + .map(outline => outline.getAttribute('xmlUrl')) + .filter((url): url is string => !!url) + .map(url => getChannelId(url)) + .filter((url): url is string => !!url) // we don't want it if it's empty +} - /** - * Reads an array of YT channel IDs from the YT subscriptions JSON file - * - * @param jsonContents a JSON file as a string - * @returns the channel IDs - */ - function readJson(jsonContents: string): string[] { - const subscriptions: YtExportedJsonSubscription[] = JSON.parse(jsonContents); +/** + * Reads an array of YT channel IDs from the YT subscriptions JSON file + * + * @param jsonContents a JSON file as a string + * @returns the channel IDs + */ +export function getSubsFromJson(jsonContents: string): string[] +{ + const subscriptions: YtExportedJsonSubscription[] = JSON.parse(jsonContents) jsonContents = '' - return subscriptions.map(sub => sub.snippet.resourceId.channelId); - } + return subscriptions.map(sub => sub.snippet.resourceId.channelId) +} - /** - * Reads an array of YT channel IDs from the YT subscriptions CSV file - * - * @param csvContent a CSV file as a string - * @returns the channel IDs - */ - function readCsv(csvContent: string): string[] { +/** + * Reads an array of YT channel IDs from the YT subscriptions CSV file + * + * @param csvContent a CSV file as a string + * @returns the channel IDs + */ +export function getSubsFromCsv(csvContent: string): string[] +{ const rows = csvContent.split('\n') csvContent = '' return rows.slice(1).map((row) => row.substring(0, row.indexOf(','))) - } +} - const URLResolverCache = (() => - { - const openRequest = self.indexedDB?.open("yt-url-resolver-cache") - - if (openRequest) - { - openRequest.addEventListener('upgradeneeded', () => openRequest.result.createObjectStore("store").createIndex("expireAt", "expireAt")) - - // Delete Expired - openRequest.addEventListener('success', () => - { - const transaction = openRequest.result.transaction("store", "readwrite") - const range = IDBKeyRange.upperBound(new Date()) - - const expireAtCursorRequest = transaction.objectStore("store").index("expireAt").openCursor(range) - expireAtCursorRequest.addEventListener('success', () => - { - const expireCursor = expireAtCursorRequest.result - if (!expireCursor) return - expireCursor.delete() - expireCursor.continue() - }) - }) - } - else console.warn(`IndexedDB not supported`) - - async function put(url: string | null, id: string): Promise - { - return await new Promise((resolve, reject) => - { - const store = openRequest.result.transaction("store", "readwrite").objectStore("store") - if (!store) return resolve() - const request = store.put({ value: url, expireAt: new Date(Date.now() + 24 * 60 * 60 * 1000) }, id) - request.addEventListener('success', () => resolve()) - request.addEventListener('error', () => reject(request.error)) - }) - } - async function get(id: string): Promise - { - return (await new Promise((resolve, reject) => - { - const store = openRequest.result.transaction("store", "readonly").objectStore("store") - if (!store) return resolve(null) - const request = store.get(id) - request.addEventListener('success', () => resolve(request.result)) - request.addEventListener('error', () => reject(request.error)) - }) as any)?.value - } - - return { put, get } - })() - - - /** - * @param descriptorsWithIndex YT resource IDs to check - * @returns a promise with the list of channels that were found on lbry - */ - async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]> { - const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({...descriptor, index})) +/** +* @param descriptorsWithIndex YT resource IDs to check +* @returns a promise with the list of channels that were found on lbry +*/ +export async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]> +{ + const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({ ...descriptor, index })) descriptors = null as any const results: (string | null)[] = [] - await Promise.all(descriptorsWithIndex.map(async (descriptor, index) => { - if (!descriptor) return - const cache = await URLResolverCache.get(descriptor.id) + await Promise.all(descriptorsWithIndex.map(async (descriptor, index) => + { + if (!descriptor) return + const cache = await LbryURLCache.get(descriptor.id) - // Cache can be null, if there is no lbry url yet - if (cache !== undefined) { - // Directly setting it to results - results[index] = cache + // Cache can be null, if there is no lbry url yet + if (cache !== undefined) + { + // Directly setting it to results + results[index] = cache - // We remove it so we dont ask it to API - descriptorsWithIndex.splice(index, 1) - } + // We remove it so we dont ask it to API + descriptorsWithIndex.splice(index, 1) + } })) - const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE); - let progressCount = 0; + const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE) + let progressCount = 0 await Promise.all(descriptorsChunks.map(async (descriptorChunk) => { - const descriptorsGroupedByType: Record = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any; + const descriptorsGroupedByType: Record = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any - const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync() - const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName] + const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync() + const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName] - const url = new URL(`https://${urlResolverSetting.hostname}`); + const url = new URL(`https://${urlResolverSetting.hostname}`) - function followResponsePath(response: any, responsePath: YtUrlResolveResponsePath) - { - for (const path of responsePath) + function followResponsePath(response: any, responsePath: YtUrlResolveResponsePath) { - switch (typeof path) - { - case 'string': - case 'number': - response = response[path] - break - default: - switch (path) - { - case Keys: - response = Object.keys(response) - break - case Values: - response = Object.values(response) - break - } - } - } - return response as T - } - - async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsWithIndex) - { - url.pathname = urlResolverFunction.pathname - - if (urlResolverFunction.paramArraySeperator === SingleValueAtATime) - { - await Promise.all(descriptorsGroup.map(async (descriptor) => { - switch (null) + for (const path of responsePath) { - default: - if (!descriptor.id) break - url.searchParams.set(urlResolverFunction.paramName, descriptor.id) - - const apiResponse = await fetch(url.toString(), { cache: 'no-store' }); - if (!apiResponse.ok) { - // Some API might not respond with 200 if it can't find the url - if (apiResponse.status === 404) await URLResolverCache.put(null, descriptor.id) - break - } - - const value = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) - if (value) results[descriptor.index] = value - await URLResolverCache.put(value, descriptor.id) + switch (typeof path) + { + case 'string': + case 'number': + response = response[path] + break + default: + switch (path) + { + case Keys: + response = Object.keys(response) + break + case Values: + response = Object.values(response) + break + } + } } - progressCount++ - if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length) - })) + return response as T } - else + + async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsWithIndex) { + url.pathname = urlResolverFunction.pathname - switch (null) - { - default: - url.searchParams - .set(urlResolverFunction.paramName, descriptorsGroup - .map((descriptor) => descriptor.id) - .filter((descriptorId) => descriptorId) - .join(urlResolverFunction.paramArraySeperator) - ) + if (urlResolverFunction.paramArraySeperator === SingleValueAtATime) + { + await Promise.all(descriptorsGroup.map(async (descriptor) => + { + switch (null) + { + default: + if (!descriptor.id) break + url.searchParams.set(urlResolverFunction.paramName, descriptor.id) - const apiResponse = await fetch(url.toString(), { cache: 'no-store' }); - if (!apiResponse.ok) break - const values = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) + const apiResponse = await fetch(url.toString(), { cache: 'no-store' }) + if (!apiResponse.ok) + { + // Some API might not respond with 200 if it can't find the url + if (apiResponse.status === 404) await LbryURLCache.put(null, descriptor.id) + break + } - await Promise.all(values.map(async (value, index) => { - const descriptor = descriptorsGroup[index] - if (value) results[descriptor.index] = value - await URLResolverCache.put(value, descriptor.id) - })) - } - progressCount += descriptorsGroup.length - if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length) + const value = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) + if (value) results[descriptor.index] = value + await LbryURLCache.put(value, descriptor.id) + } + progressCount++ + if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length) + })) + } + else + { + + switch (null) + { + default: + url.searchParams + .set(urlResolverFunction.paramName, descriptorsGroup + .map((descriptor) => descriptor.id) + .filter((descriptorId) => descriptorId) + .join(urlResolverFunction.paramArraySeperator) + ) + + const apiResponse = await fetch(url.toString(), { cache: 'no-store' }) + if (!apiResponse.ok) break + const values = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) + + await Promise.all(values.map(async (value, index) => + { + const descriptor = descriptorsGroup[index] + if (value) results[descriptor.index] = value + await LbryURLCache.put(value, descriptor.id) + })) + } + progressCount += descriptorsGroup.length + if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length) + } } - } - if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel']) - if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video']) + if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel']) + if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video']) + + })) - })); - return results - } - - return { readCsv, readJson, readOpml, resolveById } -})() +} \ No newline at end of file diff --git a/src/common/yt/urlCache.ts b/src/common/yt/urlCache.ts new file mode 100644 index 0000000..b598340 --- /dev/null +++ b/src/common/yt/urlCache.ts @@ -0,0 +1,50 @@ + +const openRequest = self.indexedDB?.open("yt-url-resolver-cache") + +if (openRequest) +{ + openRequest.addEventListener('upgradeneeded', () => openRequest.result.createObjectStore("store").createIndex("expireAt", "expireAt")) + + // Delete Expired + openRequest.addEventListener('success', () => + { + const transaction = openRequest.result.transaction("store", "readwrite") + const range = IDBKeyRange.upperBound(new Date()) + + const expireAtCursorRequest = transaction.objectStore("store").index("expireAt").openCursor(range) + expireAtCursorRequest.addEventListener('success', () => + { + const expireCursor = expireAtCursorRequest.result + if (!expireCursor) return + expireCursor.delete() + expireCursor.continue() + }) + }) +} +else console.warn(`IndexedDB not supported`) + +async function put(url: string | null, id: string): Promise +{ + return await new Promise((resolve, reject) => + { + const store = openRequest.result.transaction("store", "readwrite").objectStore("store") + if (!store) return resolve() + const request = store.put({ value: url, expireAt: new Date(Date.now() + 24 * 60 * 60 * 1000) }, id) + request.addEventListener('success', () => resolve()) + request.addEventListener('error', () => reject(request.error)) + }) +} +async function get(id: string): Promise +{ + return (await new Promise((resolve, reject) => + { + const store = openRequest.result.transaction("store", "readonly").objectStore("store") + if (!store) return resolve(null) + const request = store.get(id) + request.addEventListener('success', () => resolve(request.result)) + request.addEventListener('error', () => reject(request.error)) + }) as any)?.value +} + +export const LbryURLCache = { put, get } + diff --git a/src/scripts/ytContent.tsx b/src/scripts/ytContent.tsx index 6fb60c3..7ae21a5 100644 --- a/src/scripts/ytContent.tsx +++ b/src/scripts/ytContent.tsx @@ -1,7 +1,7 @@ -import { ExtensionSettings, getExtensionSettingsAsync, getSourcePlatfromSettingsFromHostname, TargetPlatformName, targetPlatformSettings } from '../common/settings' -import type { UpdateContext } from '../scripts/tabOnUpdated' import { h, JSX, render } from 'preact' -import { YtIdResolverDescriptor, ytService } from '../common/yt' +import { getExtensionSettingsAsync, getSourcePlatfromSettingsFromHostname, TargetPlatformName, targetPlatformSettings } from '../common/settings' +import { resolveById, YtIdResolverDescriptor } from '../common/yt' +import type { UpdateContext } from '../scripts/tabOnUpdated' const sleep = (t: number) => new Promise(resolve => setTimeout(resolve, t)); @@ -177,7 +177,7 @@ window.addEventListener('load', async () => const videoId = url.searchParams.get('v') if (!videoId) return const descriptor: YtIdResolverDescriptor = { id: videoId, type: 'video' } - const lbryPathname = (await ytService.resolveById([descriptor]))[0] + const lbryPathname = (await resolveById([descriptor]))[0] if (!lbryPathname) return updateButton({ descriptor, lbryPathname, redirect: settings.redirect, targetPlatform: settings.targetPlatform }) } diff --git a/src/tools/YTtoLBRY.tsx b/src/tools/YTtoLBRY.tsx index ac38427..7ebbe01 100644 --- a/src/tools/YTtoLBRY.tsx +++ b/src/tools/YTtoLBRY.tsx @@ -1,7 +1,7 @@ import { h, render } from 'preact' import { useState } from 'preact/hooks' import { getExtensionSettingsAsync, targetPlatformSettings } from '../common/settings' -import { getFileContent, ytService } from '../common/yt' +import { getFileContent, getSubsFromCsv, getSubsFromJson, getSubsFromOpml, resolveById } from '../common/yt' import readme from './README.md' @@ -16,10 +16,10 @@ async function lbryChannelsFromFile(file: File) { const ext = file.name.split('.').pop()?.toLowerCase(); const ids = new Set(( - ext === 'xml' || ext == 'opml' ? ytService.readOpml : - ext === 'csv' ? ytService.readCsv : - ytService.readJson)(await getFileContent(file))) - const lbryUrls = await ytService.resolveById( + ext === 'xml' || ext == 'opml' ? getSubsFromOpml : + ext === 'csv' ? getSubsFromCsv : + getSubsFromJson)(await getFileContent(file))) + const lbryUrls = await resolveById( Array.from(ids).map(id => ({ id, type: 'channel' } as const)), (progress) => render(, document.getElementById('root')!)); const { targetPlatform: platform } = await getExtensionSettingsAsync();