From 3e60ed295f3eb13e4329bdd8272a8e7abc98ae43 Mon Sep 17 00:00:00 2001 From: Shiba <44804845+DeepDoge@users.noreply.github.com> Date: Thu, 14 Apr 2022 20:58:47 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=8D=A3=20removed=20scrap=20and=20simplifi?= =?UTF-8?q?ed=20the=20urlResolver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/settings.ts | 155 ++++++++++++------------------------ src/common/yt/auth.ts | 32 ++++++++ src/common/yt/urlResolve.ts | 139 +++++++++++++------------------- src/manifest.json | 2 +- src/scripts/background.ts | 6 +- src/tools/YTtoLBRY.tsx | 6 +- 6 files changed, 147 insertions(+), 193 deletions(-) create mode 100644 src/common/yt/auth.ts diff --git a/src/common/settings.ts b/src/common/settings.ts index c6fa00c..5eb0210 100644 --- a/src/common/settings.ts +++ b/src/common/settings.ts @@ -13,8 +13,8 @@ export function getExtensionSettingsAsync(): Promise { } -export type TargetPlatformName = 'madiator.com' | 'odysee' | 'app' -export interface TargetPlatform { + +const targetPlatform = (o: { domainPrefix: string displayName: string theme: string @@ -27,10 +27,14 @@ export interface TargetPlatform { button?: JSX.CSSProperties } } +}) => o +export type TargetPlatform = ReturnType +export type TargetPlatformName = Extract +export const getTargetPlatfromSettingsEntiries = () => { + return Object.entries(targetPlatformSettings) as any as [Extract, TargetPlatform][] } - -export const targetPlatformSettings: Record = { - 'madiator.com': { +export const targetPlatformSettings = { + 'madiator.com': targetPlatform({ domainPrefix: 'https://madiator.com/', displayName: 'Madiator.com', theme: '#075656', @@ -42,8 +46,8 @@ export const targetPlatformSettings: Record icon: { transform: 'scale(1.2)' } } } - }, - odysee: { + }), + odysee: targetPlatform({ domainPrefix: 'https://odysee.com/', displayName: 'Odysee', theme: '#1e013b', @@ -51,8 +55,8 @@ export const targetPlatformSettings: Record text: 'Watch on Odysee', icon: chrome.runtime.getURL('icons/lbry/odysee-logo.svg') } - }, - app: { + }), + app: targetPlatform({ domainPrefix: 'lbry://', displayName: 'LBRY App', theme: '#075656', @@ -60,118 +64,59 @@ export const targetPlatformSettings: Record text: 'Watch on LBRY', icon: chrome.runtime.getURL('icons/lbry/lbry-logo.svg') } - }, -} - -export const getTargetPlatfromSettingsEntiries = () => { - return Object.entries(targetPlatformSettings) as any as [Extract, TargetPlatform][] + }), } -export type SourcePlatformName = 'youtube.com' | 'yewtu.be' -export interface SourcePlatform { + + +const sourcePlatform = (o: { hostnames: string[] htmlQueries: { mountButtonBefore: string, videoPlayer: string } -} - -export const sourcePlatfromSettings: Record = { - "yewtu.be": { - hostnames: ['yewtu.be', 'vid.puffyan.us', 'invidio.xamh.de', 'invidious.kavin.rocks'], - htmlQueries: { - mountButtonBefore: '#watch-on-youtube', - videoPlayer: '#player-container video' - } - }, - "youtube.com": { - hostnames: ['www.youtube.com'], - htmlQueries: { - mountButtonBefore: 'ytd-video-owner-renderer~#subscribe-button', - videoPlayer: '#ytd-player video' - } - } -} - +}) => o +export type SourcePlatform = ReturnType +export type SourcePlatformName = Extract export function getSourcePlatfromSettingsFromHostname(hostname: string) { const values = Object.values(sourcePlatfromSettings) for (const settings of values) if (settings.hostnames.includes(hostname)) return settings return null } - - -export type YTUrlResolverName = 'lbryInc' | 'madiatorScrap' - -export const Keys = Symbol('keys') -export const Values = Symbol('values') -export const SingleValueAtATime = Symbol() -export type YtUrlResolveResponsePath = (string | number | typeof Keys | typeof Values)[] -export interface YtUrlResolveFunction { - pathname: string - defaultParams: Record - valueParamName: string - paramArraySeperator: string | typeof SingleValueAtATime - responsePath: YtUrlResolveResponsePath +export const sourcePlatfromSettings = { + "yewtu.be": sourcePlatform({ + hostnames: ['yewtu.be', 'vid.puffyan.us', 'invidio.xamh.de', 'invidious.kavin.rocks'], + htmlQueries: { + mountButtonBefore: '#watch-on-youtube', + videoPlayer: '#player-container video' + } + }), + "youtube.com": sourcePlatform({ + hostnames: ['www.youtube.com'], + htmlQueries: { + mountButtonBefore: 'ytd-video-owner-renderer~#subscribe-button', + videoPlayer: '#ytd-player video' + } + }) } -export interface YTUrlResolver { + +const ytUrlResolver = (o: { name: string - hostname: string - functions: { - getChannelId: YtUrlResolveFunction - getVideoId: YtUrlResolveFunction - } -} - -export const ytUrlResolversSettings: Record = { - lbryInc: { + href: string +}) => o +export type YTUrlResolver = ReturnType +export type YTUrlResolverName = Extract +export const getYtUrlResolversSettingsEntiries = () => Object.entries(ytUrlResolversSettings) as any as [Extract, YTUrlResolver][] +export const ytUrlResolversSettings = { + lbryInc: ytUrlResolver({ name: "Odysee", - hostname: "api.odysee.com", - functions: { - getChannelId: { - pathname: "/yt/resolve", - defaultParams: {}, - valueParamName: "channel_ids", - paramArraySeperator: ',', - responsePath: ["data", "channels", Values] - }, - getVideoId: { - pathname: "/yt/resolve", - defaultParams: {}, - valueParamName: "video_ids", - paramArraySeperator: ",", - responsePath: ["data", "videos", Values] - } - } - }, - madiatorScrap: { - name: "Madiator.com", - hostname: "scrap.madiator.com", - functions: { - getChannelId: { - pathname: "/api/get-lbry-channel", - defaultParams: { - v: 2 - }, - valueParamName: "url", - paramArraySeperator: SingleValueAtATime, - responsePath: ["lbrych"] - }, - getVideoId: { - pathname: "/api/get-lbry-video", - defaultParams: { - v: 2 - }, - valueParamName: "url", - paramArraySeperator: SingleValueAtATime, - responsePath: ["lbryurl"] - } - } - } -} - -export const getYtUrlResolversSettingsEntiries = () => { - return Object.entries(ytUrlResolversSettings) as any as [Extract, YTUrlResolver][] + href: "https://api.odysee.com/yt/resolve" + }), + madiatorFinder: ytUrlResolver({ + name: "Madiator Finder", + href: "https://finder.madiator.com/api/v1/resolve" + }) } \ No newline at end of file diff --git a/src/common/yt/auth.ts b/src/common/yt/auth.ts new file mode 100644 index 0000000..32ea04f --- /dev/null +++ b/src/common/yt/auth.ts @@ -0,0 +1,32 @@ +import crypto from 'crypto' + +function generateKeys() { + // The `generateKeyPairSync` method accepts two arguments: + // 1. The type ok keys we want, which in this case is "rsa" + // 2. An object with the properties of the key + const keys = crypto.generateKeyPairSync("rsa", { + // The standard secure default length for RSA keys is 2048 bits + modulusLength: 2048, + }) + + return keys +} + +function encodeKeys(keys: { publicKey: Buffer, privateKey: Buffer }) { + return JSON.stringify({ publicKey: keys.publicKey.toString('base64'), privateKey: keys.privateKey.toString('base64') }) +} + +function decodeKeys(encodedKeys: string) { + const keysBase64 = JSON.parse(encodedKeys) + return { + publicKey: Buffer.from(keysBase64.publicKey), + privateKey: Buffer.from(keysBase64.privateKey) + } +} + +function sign(data: string, privateKey: Buffer) { + return crypto.sign("sha256", Buffer.from(data), { + key: privateKey, + padding: crypto.constants.RSA_PKCS1_PSS_PADDING, + }) +} \ No newline at end of file diff --git a/src/common/yt/urlResolve.ts b/src/common/yt/urlResolve.ts index d02e3d4..559b52c 100644 --- a/src/common/yt/urlResolve.ts +++ b/src/common/yt/urlResolve.ts @@ -1,105 +1,80 @@ -import { chunk, groupBy } from "lodash" -import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YtUrlResolveResponsePath, ytUrlResolversSettings } from "../settings" +import { chunk } from "lodash" +import { getExtensionSettingsAsync, ytUrlResolversSettings } from "../settings" import { LbryPathnameCache } from "./urlCache" // const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS const QUERY_CHUNK_SIZE = 100 -export interface YtIdResolverDescriptor { - id: string - type: 'channel' | 'video' + +export type YtUrlResolveItem = { type: 'video' | 'channel', id: string } +type Results = Record +type Paramaters = YtUrlResolveItem[] + +interface ApiResponse { + channels?: Record + videos?: Record } -/** -* @param descriptorsWithIndex YT resource IDs to check -* @returns a promise with the list of channels that were found on lbry -*/ -export async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]> { - let descriptorsPayload: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({ ...descriptor, index })) - descriptors = null as any - const results: (string | null)[] = [] +export async function resolveById(params: Paramaters, progressCallback?: (progress: number) => void): Promise { + const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync() + const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName] + async function requestChunk(params: Paramaters) { + const results: Results = {} - descriptorsPayload = (await Promise.all(descriptorsPayload.map(async (descriptor, index) => { - if (!descriptor?.id) return - const cache = await LbryPathnameCache.get(descriptor.id) + // Check for cache first, add them to the results if there are any cache + // And remove them from the params, so we dont request for them + params = (await Promise.all(params.map(async (item) => { + const cachedLbryUrl = await LbryPathnameCache.get(item.id) - // Cache can be null, if there is no lbry url yet - if (cache !== undefined) { - // Null values shouldn't be in the results - if (cache) results[index] = cache - return - } - - return descriptor - }))).filter((descriptor) => descriptor) as any - - const descriptorsPayloadChunks = chunk(descriptorsPayload, QUERY_CHUNK_SIZE) - let progressCount = 0 - await Promise.all(descriptorsPayloadChunks.map(async (descriptorChunk) => { - const descriptorsGroupedByType: Record = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any - - const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync() - const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName] - - const url = new URL(`https://${urlResolverSetting.hostname}`) - - function followResponsePath(response: any, responsePath: YtUrlResolveResponsePath) { - for (const path of responsePath) { - switch (typeof path) { - case 'string': case 'number': response = response[path]; continue - } - switch (path) { - case Keys: response = Object.keys(response); continue - case Values: response = Object.values(response); continue - } + // Cache can be null, if there is no lbry url yet + if (cachedLbryUrl !== undefined) { + // Null values shouldn't be in the results + if (cachedLbryUrl !== null) results[item.id] = { id: cachedLbryUrl, type: item.type } + return null } - return response as T - } - async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsPayload) { - url.pathname = urlResolverFunction.pathname - Object.entries(urlResolverFunction.defaultParams).forEach(([name, value]) => url.searchParams.set(name, value.toString())) + // No cache found + return item + }))).filter((o) => o) as Paramaters - if (urlResolverFunction.paramArraySeperator === SingleValueAtATime) { - await Promise.all(descriptorsGroup.map(async (descriptor) => { - url.searchParams.set(urlResolverFunction.valueParamName, descriptor.id) + if (params.length === 0) return results - const apiResponse = await fetch(url.toString(), { cache: 'no-store' }) - if (apiResponse.ok) { - const value = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) - if (value) results[descriptor.index] = value - await LbryPathnameCache.put(value, descriptor.id) - } - else if (apiResponse.status === 404) await LbryPathnameCache.put(null, descriptor.id) + const url = new URL(`${urlResolverSetting.href}`) + url.searchParams.set('video_ids', params.filter((item) => item.type === 'video').map((item) => item.id).join(',')) + url.searchParams.set('channel_ids', params.filter((item) => item.type === 'channel').map((item) => item.id).join(',')) - progressCount++ - if (progressCallback) progressCallback(progressCount / descriptorsPayload.length) - })) + const apiResponse = await fetch(url.toString(), { cache: 'no-store' }) + if (apiResponse.ok) { + const response: ApiResponse = await apiResponse.json() + for (const [id, lbryUrl] of Object.entries(response.channels ?? {})) { + // we cache it no matter if its null or not + await LbryPathnameCache.put(lbryUrl, id) + + if (lbryUrl) results[id] = { id: lbryUrl, type: 'channel' } } - else { - url.searchParams.set(urlResolverFunction.valueParamName, descriptorsGroup - .map((descriptor) => descriptor.id) - .join(urlResolverFunction.paramArraySeperator)) + for (const [id, lbryUrl] of Object.entries(response.videos ?? {})) { + // we cache it no matter if its null or not + await LbryPathnameCache.put(lbryUrl, id) - const apiResponse = await fetch(url.toString(), { cache: 'no-store' }) - if (apiResponse.ok) { - const values = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) - await Promise.all(values.map(async (value, index) => { - const descriptor = descriptorsGroup[index] - if (value) results[descriptor.index] = value - await LbryPathnameCache.put(value, descriptor.id) - })) - } - - progressCount += descriptorsGroup.length - if (progressCallback) progressCallback(progressCount / descriptorsPayload.length) + if (lbryUrl) results[id] = { id: lbryUrl, type: 'video' } } } - if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel']) - if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video']) - })) + return results + } + + const results: Results = {} + const chunks = chunk(params, QUERY_CHUNK_SIZE) + + let i = 0 + if (progressCallback) progressCallback(0) + for (const chunk of chunks) { + if (progressCallback) progressCallback(++i / (chunks.length + 1)) + Object.assign(results, await requestChunk(chunk)) + } + console.log(results) + if (progressCallback) progressCallback(1) return results } diff --git a/src/manifest.json b/src/manifest.json index 891ccf8..343d3f3 100644 --- a/src/manifest.json +++ b/src/manifest.json @@ -11,7 +11,7 @@ "https://lbry.tv/", "https://odysee.com/", "https://madiator.com/", - "https://scrap.madiator.com/", + "https://finder.madiator.com/", "tabs", "storage" ], diff --git a/src/scripts/background.ts b/src/scripts/background.ts index c30af4f..5fe7a4a 100644 --- a/src/scripts/background.ts +++ b/src/scripts/background.ts @@ -1,7 +1,7 @@ -import { resolveById, YtIdResolverDescriptor } from '../common/yt/urlResolve' +import { resolveById, YtUrlResolveItem } from '../common/yt/urlResolve' -async function resolveYT(descriptor: YtIdResolverDescriptor) { - const lbryProtocolUrl: string | null = (await resolveById([descriptor]).then(a => a[0])) ?? null +async function resolveYT(item: YtUrlResolveItem) { + const lbryProtocolUrl: string | null = (await resolveById([item]).then((items) => items[item.id]))?.id ?? null if (!lbryProtocolUrl) return null return lbryProtocolUrl.replaceAll('#', ':') /* const segments = parseProtocolUrl(lbryProtocolUrl || '', { encode: true }) diff --git a/src/tools/YTtoLBRY.tsx b/src/tools/YTtoLBRY.tsx index 5faaa29..8709429 100644 --- a/src/tools/YTtoLBRY.tsx +++ b/src/tools/YTtoLBRY.tsx @@ -18,12 +18,14 @@ async function lbryChannelsFromFile(file: File) { ext === 'xml' || ext == 'opml' ? getSubsFromOpml : ext === 'csv' ? getSubsFromCsv : getSubsFromJson)(await getFileContent(file))) - const lbryPathnames = await resolveById( + + const items = await resolveById( Array.from(ids).map(id => ({ id, type: 'channel' } as const)), (progress) => render(, document.getElementById('root')!)) + const { targetPlatform: platform } = await getExtensionSettingsAsync() const urlPrefix = targetPlatformSettings[platform].domainPrefix - return lbryPathnames.map(channel => urlPrefix + channel) + return Object.values(items).map((item) => urlPrefix + item.id) } function ConversionCard({ onSelect, progress }: { onSelect(file: File): Promise | void, progress: number }) {