From 2b914369003c6af26c8bc012481c859fbf7f8fa5 Mon Sep 17 00:00:00 2001 From: Shiba <44804845+DeepDoge@users.noreply.github.com> Date: Sun, 9 Jan 2022 21:11:58 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=A5=A1=20Refactor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/yt/index.ts | 229 +++++++----------------------------- src/common/yt/urlResolve.ts | 141 ++++++++++++++++++++++ src/scripts/background.ts | 2 +- src/tools/YTtoLBRY.tsx | 3 +- 4 files changed, 188 insertions(+), 187 deletions(-) create mode 100644 src/common/yt/urlResolve.ts diff --git a/src/common/yt/index.ts b/src/common/yt/index.ts index 31f0440..58a7859 100644 --- a/src/common/yt/index.ts +++ b/src/common/yt/index.ts @@ -1,10 +1,3 @@ -import chunk from 'lodash/chunk' -import groupBy from 'lodash/groupBy' -import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YtUrlResolveResponsePath, ytUrlResolversSettings } from '../settings' -import { LbryPathnameCache } from './urlCache' - -// const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS -const QUERY_CHUNK_SIZE = 300 interface YtExportedJsonSubscription { @@ -19,11 +12,6 @@ interface YtExportedJsonSubscription } } -export interface YtIdResolverDescriptor -{ - id: string - type: 'channel' | 'video' -} /** * @param file to load @@ -44,6 +32,50 @@ export function getFileContent(file: File): Promise }) } + +/** + * Reads the array of YT channels from an OPML file + * + * @param opmlContents an opml file as as tring + * @returns the channel IDs + */ + export function getSubsFromOpml(opmlContents: string): string[] + { + const opml = new DOMParser().parseFromString(opmlContents, 'application/xml') + opmlContents = '' + return Array.from(opml.querySelectorAll('outline > outline')) + .map(outline => outline.getAttribute('xmlUrl')) + .filter((url): url is string => !!url) + .map(url => getChannelId(url)) + .filter((url): url is string => !!url) // we don't want it if it's empty + } + + /** + * Reads an array of YT channel IDs from the YT subscriptions JSON file + * + * @param jsonContents a JSON file as a string + * @returns the channel IDs + */ + export function getSubsFromJson(jsonContents: string): string[] + { + const subscriptions: YtExportedJsonSubscription[] = JSON.parse(jsonContents) + jsonContents = '' + return subscriptions.map(sub => sub.snippet.resourceId.channelId) + } + + /** + * Reads an array of YT channel IDs from the YT subscriptions CSV file + * + * @param csvContent a CSV file as a string + * @returns the channel IDs + */ + export function getSubsFromCsv(csvContent: string): string[] + { + const rows = csvContent.split('\n') + csvContent = '' + return rows.slice(1).map((row) => row.substring(0, row.indexOf(','))) + } + /** * Extracts the channelID from a YT URL. * @@ -77,177 +109,4 @@ export function parseYouTubeURLTimeString(timeString: string) total += t } return total.toString() -} - - -/** - * Reads the array of YT channels from an OPML file - * - * @param opmlContents an opml file as as tring - * @returns the channel IDs - */ -export function getSubsFromOpml(opmlContents: string): string[] -{ - const opml = new DOMParser().parseFromString(opmlContents, 'application/xml') - opmlContents = '' - return Array.from(opml.querySelectorAll('outline > outline')) - .map(outline => outline.getAttribute('xmlUrl')) - .filter((url): url is string => !!url) - .map(url => getChannelId(url)) - .filter((url): url is string => !!url) // we don't want it if it's empty -} - -/** - * Reads an array of YT channel IDs from the YT subscriptions JSON file - * - * @param jsonContents a JSON file as a string - * @returns the channel IDs - */ -export function getSubsFromJson(jsonContents: string): string[] -{ - const subscriptions: YtExportedJsonSubscription[] = JSON.parse(jsonContents) - jsonContents = '' - return subscriptions.map(sub => sub.snippet.resourceId.channelId) -} - -/** - * Reads an array of YT channel IDs from the YT subscriptions CSV file - * - * @param csvContent a CSV file as a string - * @returns the channel IDs - */ -export function getSubsFromCsv(csvContent: string): string[] -{ - const rows = csvContent.split('\n') - csvContent = '' - return rows.slice(1).map((row) => row.substring(0, row.indexOf(','))) -} - -/** -* @param descriptorsWithIndex YT resource IDs to check -* @returns a promise with the list of channels that were found on lbry -*/ -export async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]> -{ - const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({ ...descriptor, index })) - descriptors = null as any - const results: (string | null)[] = [] - - await Promise.all(descriptorsWithIndex.map(async (descriptor, index) => - { - if (!descriptor) return - const cache = await LbryPathnameCache.get(descriptor.id) - - // Cache can be null, if there is no lbry url yet - if (cache !== undefined) - { - // Directly setting it to results - results[index] = cache - - // We remove it so we dont ask it to API - descriptorsWithIndex.splice(index, 1) - } - })) - - const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE) - let progressCount = 0 - await Promise.all(descriptorsChunks.map(async (descriptorChunk) => - { - const descriptorsGroupedByType: Record = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any - - const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync() - const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName] - - const url = new URL(`https://${urlResolverSetting.hostname}`) - - function followResponsePath(response: any, responsePath: YtUrlResolveResponsePath) - { - for (const path of responsePath) - { - switch (typeof path) - { - case 'string': - case 'number': - response = response[path] - break - default: - switch (path) - { - case Keys: - response = Object.keys(response) - break - case Values: - response = Object.values(response) - break - } - } - } - return response as T - } - - async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsWithIndex) - { - url.pathname = urlResolverFunction.pathname - - if (urlResolverFunction.paramArraySeperator === SingleValueAtATime) - { - await Promise.all(descriptorsGroup.map(async (descriptor) => - { - switch (null) - { - default: - if (!descriptor.id) break - url.searchParams.set(urlResolverFunction.paramName, descriptor.id) - - const apiResponse = await fetch(url.toString(), { cache: 'no-store' }) - if (!apiResponse.ok) - { - // Some API might not respond with 200 if it can't find the url - if (apiResponse.status === 404) await LbryPathnameCache.put(null, descriptor.id) - break - } - - const value = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) - if (value) results[descriptor.index] = value - await LbryPathnameCache.put(value, descriptor.id) - } - progressCount++ - if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length) - })) - } - else - { - - switch (null) - { - default: - url.searchParams - .set(urlResolverFunction.paramName, descriptorsGroup - .map((descriptor) => descriptor.id) - .filter((descriptorId) => descriptorId) - .join(urlResolverFunction.paramArraySeperator) - ) - - const apiResponse = await fetch(url.toString(), { cache: 'no-store' }) - if (!apiResponse.ok) break - const values = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) - - await Promise.all(values.map(async (value, index) => - { - const descriptor = descriptorsGroup[index] - if (value) results[descriptor.index] = value - await LbryPathnameCache.put(value, descriptor.id) - })) - } - progressCount += descriptorsGroup.length - if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length) - } - } - - if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel']) - if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video']) - - })) - - return results } \ No newline at end of file diff --git a/src/common/yt/urlResolve.ts b/src/common/yt/urlResolve.ts new file mode 100644 index 0000000..26f45da --- /dev/null +++ b/src/common/yt/urlResolve.ts @@ -0,0 +1,141 @@ +import { chunk, groupBy } from "lodash" +import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YtUrlResolveResponsePath, ytUrlResolversSettings } from "../settings" +import { LbryPathnameCache } from "./urlCache" + +// const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS +const QUERY_CHUNK_SIZE = 300 + +export interface YtIdResolverDescriptor +{ + id: string + type: 'channel' | 'video' +} + + /** + * @param descriptorsWithIndex YT resource IDs to check + * @returns a promise with the list of channels that were found on lbry + */ + export async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]> + { + const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({ ...descriptor, index })) + descriptors = null as any + const results: (string | null)[] = [] + + await Promise.all(descriptorsWithIndex.map(async (descriptor, index) => + { + if (!descriptor) return + const cache = await LbryPathnameCache.get(descriptor.id) + + // Cache can be null, if there is no lbry url yet + if (cache !== undefined) + { + // Directly setting it to results + results[index] = cache + + // We remove it so we dont ask it to API + descriptorsWithIndex.splice(index, 1) + } + })) + + const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE) + let progressCount = 0 + await Promise.all(descriptorsChunks.map(async (descriptorChunk) => + { + const descriptorsGroupedByType: Record = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any + + const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync() + const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName] + + const url = new URL(`https://${urlResolverSetting.hostname}`) + + function followResponsePath(response: any, responsePath: YtUrlResolveResponsePath) + { + for (const path of responsePath) + { + switch (typeof path) + { + case 'string': + case 'number': + response = response[path] + break + default: + switch (path) + { + case Keys: + response = Object.keys(response) + break + case Values: + response = Object.values(response) + break + } + } + } + return response as T + } + + async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsWithIndex) + { + url.pathname = urlResolverFunction.pathname + + if (urlResolverFunction.paramArraySeperator === SingleValueAtATime) + { + await Promise.all(descriptorsGroup.map(async (descriptor) => + { + switch (null) + { + default: + if (!descriptor.id) break + url.searchParams.set(urlResolverFunction.paramName, descriptor.id) + + const apiResponse = await fetch(url.toString(), { cache: 'no-store' }) + if (!apiResponse.ok) + { + // Some API might not respond with 200 if it can't find the url + if (apiResponse.status === 404) await LbryPathnameCache.put(null, descriptor.id) + break + } + + const value = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) + if (value) results[descriptor.index] = value + await LbryPathnameCache.put(value, descriptor.id) + } + progressCount++ + if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length) + })) + } + else + { + + switch (null) + { + default: + url.searchParams + .set(urlResolverFunction.paramName, descriptorsGroup + .map((descriptor) => descriptor.id) + .filter((descriptorId) => descriptorId) + .join(urlResolverFunction.paramArraySeperator) + ) + + const apiResponse = await fetch(url.toString(), { cache: 'no-store' }) + if (!apiResponse.ok) break + const values = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) + + await Promise.all(values.map(async (value, index) => + { + const descriptor = descriptorsGroup[index] + if (value) results[descriptor.index] = value + await LbryPathnameCache.put(value, descriptor.id) + })) + } + progressCount += descriptorsGroup.length + if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length) + } + } + + if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel']) + if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video']) + + })) + + return results + } \ No newline at end of file diff --git a/src/scripts/background.ts b/src/scripts/background.ts index a86ef0e..591c7e2 100644 --- a/src/scripts/background.ts +++ b/src/scripts/background.ts @@ -1,5 +1,5 @@ import { parseProtocolUrl } from '../common/lbry-url' -import { resolveById, YtIdResolverDescriptor } from '../common/yt' +import { resolveById, YtIdResolverDescriptor } from '../common/yt/urlResolve' async function resolveYT(descriptor: YtIdResolverDescriptor) { const lbryProtocolUrl: string | null = await resolveById([descriptor]).then(a => a[0]); const segments = parseProtocolUrl(lbryProtocolUrl || '', { encode: true }); diff --git a/src/tools/YTtoLBRY.tsx b/src/tools/YTtoLBRY.tsx index ab120a1..ca89dc0 100644 --- a/src/tools/YTtoLBRY.tsx +++ b/src/tools/YTtoLBRY.tsx @@ -1,7 +1,8 @@ import { h, render } from 'preact' import { useState } from 'preact/hooks' import { getExtensionSettingsAsync, targetPlatformSettings } from '../common/settings' -import { getFileContent, getSubsFromCsv, getSubsFromJson, getSubsFromOpml, resolveById } from '../common/yt' +import { getFileContent, getSubsFromCsv, getSubsFromJson, getSubsFromOpml } from '../common/yt' +import { resolveById } from '../common/yt/urlResolve' import readme from './README.md'