LBRYFoundation/Watch-on-LBRY
1
0
Fork 0
mirror of https://github.com/LBRYFoundation/Watch-on-LBRY.git synced 2025-08-23 17:47:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

🥞 Refactor

Browse source
This commit is contained in:
Shiba 2022-01-09 19:10:27 +00:00
parent 75cb9cf01d
commit cb4b4f4b2e
4 changed files with 239 additions and 230 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

401
src/common/yt/index.ts
View file

@ -1,271 +1,230 @@
import chunk from 'lodash/chunk'
import groupBy from 'lodash/groupBy'
import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YtUrlResolveResponsePath, ytUrlResolversSettings } from '../settings'
import { LbryURLCache } from './urlCache'
// const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS
const QUERY_CHUNK_SIZE = 300;
const QUERY_CHUNK_SIZE = 300
interface YtExportedJsonSubscription {
id: string;
etag: string;
title: string;
snippet: {
description: string;
resourceId: {
channelId: string;
};
};
interface YtExportedJsonSubscription
{
id: string
etag: string
title: string
snippet: {
description: string
resourceId: {
channelId: string
}
}
}
export interface YtIdResolverDescriptor {
id: string
type: 'channel' | 'video'
export interface YtIdResolverDescriptor
{
id: string
type: 'channel' | 'video'
}
/**
* @param file to load
* @returns a promise with the file as a string
*/
export function getFileContent(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.addEventListener('load', event => resolve(event.target?.result as string || ''));
reader.addEventListener('error', () => {
reader.abort();
reject(new DOMException(`Could not read ${file.name}`));
});
reader.readAsText(file);
});
export function getFileContent(file: File): Promise<string>
{
return new Promise((resolve, reject) =>
{
const reader = new FileReader()
reader.addEventListener('load', event => resolve(event.target?.result as string || ''))
reader.addEventListener('error', () =>
{
reader.abort()
reject(new DOMException(`Could not read ${file.name}`))
})
reader.readAsText(file)
})
}
export const ytService = (() => {
/**
* Extracts the channelID from a YT URL.
*
* Handles these two types of YT URLs:
* * /feeds/videos.xml?channel_id=*
* * /channel/*
*/
function getChannelId(channelURL: string) {
const match = channelURL.match(/channel\/([^\s?]*)/);
return match ? match[1] : new URL(channelURL).searchParams.get('channel_id');
}
/**
* Extracts the channelID from a YT URL.
*
* Handles these two types of YT URLs:
* * /feeds/videos.xml?channel_id=*
* * /channel/*
*/
export function getChannelId(channelURL: string)
{
const match = channelURL.match(/channel\/([^\s?]*)/)
return match ? match[1] : new URL(channelURL).searchParams.get('channel_id')
}
/**
* Reads the array of YT channels from an OPML file
*
* @param opmlContents an opml file as as tring
* @returns the channel IDs
*/
function readOpml(opmlContents: string): string[] {
const opml = new DOMParser().parseFromString(opmlContents, 'application/xml');
/**
* Reads the array of YT channels from an OPML file
*
* @param opmlContents an opml file as as tring
* @returns the channel IDs
*/
export function getSubsFromOpml(opmlContents: string): string[]
{
const opml = new DOMParser().parseFromString(opmlContents, 'application/xml')
opmlContents = ''
return Array.from(opml.querySelectorAll('outline > outline'))
.map(outline => outline.getAttribute('xmlUrl'))
.filter((url): url is string => !!url)
.map(url => getChannelId(url))
.filter((url): url is string => !!url); // we don't want it if it's empty
}
.map(outline => outline.getAttribute('xmlUrl'))
.filter((url): url is string => !!url)
.map(url => getChannelId(url))
.filter((url): url is string => !!url) // we don't want it if it's empty
}
/**
* Reads an array of YT channel IDs from the YT subscriptions JSON file
*
* @param jsonContents a JSON file as a string
* @returns the channel IDs
*/
function readJson(jsonContents: string): string[] {
const subscriptions: YtExportedJsonSubscription[] = JSON.parse(jsonContents);
/**
* Reads an array of YT channel IDs from the YT subscriptions JSON file
*
* @param jsonContents a JSON file as a string
* @returns the channel IDs
*/
export function getSubsFromJson(jsonContents: string): string[]
{
const subscriptions: YtExportedJsonSubscription[] = JSON.parse(jsonContents)
jsonContents = ''
return subscriptions.map(sub => sub.snippet.resourceId.channelId);
}
return subscriptions.map(sub => sub.snippet.resourceId.channelId)
}
/**
* Reads an array of YT channel IDs from the YT subscriptions CSV file
*
* @param csvContent a CSV file as a string
* @returns the channel IDs
*/
function readCsv(csvContent: string): string[] {
/**
* Reads an array of YT channel IDs from the YT subscriptions CSV file
*
* @param csvContent a CSV file as a string
* @returns the channel IDs
*/
export function getSubsFromCsv(csvContent: string): string[]
{
const rows = csvContent.split('\n')
csvContent = ''
return rows.slice(1).map((row) => row.substring(0, row.indexOf(',')))
}
}
const URLResolverCache = (() =>
{
const openRequest = self.indexedDB?.open("yt-url-resolver-cache")
if (openRequest)
{
openRequest.addEventListener('upgradeneeded', () => openRequest.result.createObjectStore("store").createIndex("expireAt", "expireAt"))
// Delete Expired
openRequest.addEventListener('success', () =>
{
const transaction = openRequest.result.transaction("store", "readwrite")
const range = IDBKeyRange.upperBound(new Date())
const expireAtCursorRequest = transaction.objectStore("store").index("expireAt").openCursor(range)
expireAtCursorRequest.addEventListener('success', () =>
{
const expireCursor = expireAtCursorRequest.result
if (!expireCursor) return
expireCursor.delete()
expireCursor.continue()
})
})
}
else console.warn(`IndexedDB not supported`)
async function put(url: string | null, id: string): Promise<void>
{
return await new Promise((resolve, reject) =>
{
const store = openRequest.result.transaction("store", "readwrite").objectStore("store")
if (!store) return resolve()
const request = store.put({ value: url, expireAt: new Date(Date.now() + 24 * 60 * 60 * 1000) }, id)
request.addEventListener('success', () => resolve())
request.addEventListener('error', () => reject(request.error))
})
}
async function get(id: string): Promise<string | null>
{
return (await new Promise((resolve, reject) =>
{
const store = openRequest.result.transaction("store", "readonly").objectStore("store")
if (!store) return resolve(null)
const request = store.get(id)
request.addEventListener('success', () => resolve(request.result))
request.addEventListener('error', () => reject(request.error))
}) as any)?.value
}
return { put, get }
})()
/**
* @param descriptorsWithIndex YT resource IDs to check
* @returns a promise with the list of channels that were found on lbry
*/
async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]> {
const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({...descriptor, index}))
/**
* @param descriptorsWithIndex YT resource IDs to check
* @returns a promise with the list of channels that were found on lbry
*/
export async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]>
{
const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({ ...descriptor, index }))
descriptors = null as any
const results: (string | null)[] = []
await Promise.all(descriptorsWithIndex.map(async (descriptor, index) => {
if (!descriptor) return
const cache = await URLResolverCache.get(descriptor.id)
await Promise.all(descriptorsWithIndex.map(async (descriptor, index) =>
{
if (!descriptor) return
const cache = await LbryURLCache.get(descriptor.id)
// Cache can be null, if there is no lbry url yet
if (cache !== undefined) {
// Directly setting it to results
results[index] = cache
// Cache can be null, if there is no lbry url yet
if (cache !== undefined)
{
// Directly setting it to results
results[index] = cache
// We remove it so we dont ask it to API
descriptorsWithIndex.splice(index, 1)
}
// We remove it so we dont ask it to API
descriptorsWithIndex.splice(index, 1)
}
}))
const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE);
let progressCount = 0;
const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE)
let progressCount = 0
await Promise.all(descriptorsChunks.map(async (descriptorChunk) =>
{
const descriptorsGroupedByType: Record<YtIdResolverDescriptor['type'], typeof descriptorsWithIndex | null> = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any;
const descriptorsGroupedByType: Record<YtIdResolverDescriptor['type'], typeof descriptorsWithIndex | null> = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any
const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync()
const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName]
const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync()
const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName]
const url = new URL(`https://${urlResolverSetting.hostname}`);
const url = new URL(`https://${urlResolverSetting.hostname}`)
function followResponsePath<T>(response: any, responsePath: YtUrlResolveResponsePath)
{
for (const path of responsePath)
function followResponsePath<T>(response: any, responsePath: YtUrlResolveResponsePath)
{
switch (typeof path)
{
case 'string':
case 'number':
response = response[path]
break
default:
switch (path)
{
case Keys:
response = Object.keys(response)
break
case Values:
response = Object.values(response)
break
}
}
}
return response as T
}
async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsWithIndex)
{
url.pathname = urlResolverFunction.pathname
if (urlResolverFunction.paramArraySeperator === SingleValueAtATime)
{
await Promise.all(descriptorsGroup.map(async (descriptor) => {
switch (null)
for (const path of responsePath)
{
default:
if (!descriptor.id) break
url.searchParams.set(urlResolverFunction.paramName, descriptor.id)
const apiResponse = await fetch(url.toString(), { cache: 'no-store' });
if (!apiResponse.ok) {
// Some API might not respond with 200 if it can't find the url
if (apiResponse.status === 404) await URLResolverCache.put(null, descriptor.id)
break
}
const value = followResponsePath<string>(await apiResponse.json(), urlResolverFunction.responsePath)
if (value) results[descriptor.index] = value
await URLResolverCache.put(value, descriptor.id)
switch (typeof path)
{
case 'string':
case 'number':
response = response[path]
break
default:
switch (path)
{
case Keys:
response = Object.keys(response)
break
case Values:
response = Object.values(response)
break
}
}
}
progressCount++
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
}))
return response as T
}
else
async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsWithIndex)
{
url.pathname = urlResolverFunction.pathname
switch (null)
{
default:
url.searchParams
.set(urlResolverFunction.paramName, descriptorsGroup
.map((descriptor) => descriptor.id)
.filter((descriptorId) => descriptorId)
.join(urlResolverFunction.paramArraySeperator)
)
if (urlResolverFunction.paramArraySeperator === SingleValueAtATime)
{
await Promise.all(descriptorsGroup.map(async (descriptor) =>
{
switch (null)
{
default:
if (!descriptor.id) break
url.searchParams.set(urlResolverFunction.paramName, descriptor.id)
const apiResponse = await fetch(url.toString(), { cache: 'no-store' });
if (!apiResponse.ok) break
const values = followResponsePath<string[]>(await apiResponse.json(), urlResolverFunction.responsePath)
const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
if (!apiResponse.ok)
{
// Some API might not respond with 200 if it can't find the url
if (apiResponse.status === 404) await LbryURLCache.put(null, descriptor.id)
break
}
await Promise.all(values.map(async (value, index) => {
const descriptor = descriptorsGroup[index]
if (value) results[descriptor.index] = value
await URLResolverCache.put(value, descriptor.id)
}))
}
progressCount += descriptorsGroup.length
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
const value = followResponsePath<string>(await apiResponse.json(), urlResolverFunction.responsePath)
if (value) results[descriptor.index] = value
await LbryURLCache.put(value, descriptor.id)
}
progressCount++
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
}))
}
else
{
switch (null)
{
default:
url.searchParams
.set(urlResolverFunction.paramName, descriptorsGroup
.map((descriptor) => descriptor.id)
.filter((descriptorId) => descriptorId)
.join(urlResolverFunction.paramArraySeperator)
)
const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
if (!apiResponse.ok) break
const values = followResponsePath<string[]>(await apiResponse.json(), urlResolverFunction.responsePath)
await Promise.all(values.map(async (value, index) =>
{
const descriptor = descriptorsGroup[index]
if (value) results[descriptor.index] = value
await LbryURLCache.put(value, descriptor.id)
}))
}
progressCount += descriptorsGroup.length
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
}
}
}
if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel'])
if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video'])
if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel'])
if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video'])
}))
}));
return results
}
return { readCsv, readJson, readOpml, resolveById }
})()
}

50
src/common/yt/urlCache.ts Normal file
View file

@ -0,0 +1,50 @@
const openRequest = self.indexedDB?.open("yt-url-resolver-cache")
if (openRequest)
{
openRequest.addEventListener('upgradeneeded', () => openRequest.result.createObjectStore("store").createIndex("expireAt", "expireAt"))
// Delete Expired
openRequest.addEventListener('success', () =>
{
const transaction = openRequest.result.transaction("store", "readwrite")
const range = IDBKeyRange.upperBound(new Date())
const expireAtCursorRequest = transaction.objectStore("store").index("expireAt").openCursor(range)
expireAtCursorRequest.addEventListener('success', () =>
{
const expireCursor = expireAtCursorRequest.result
if (!expireCursor) return
expireCursor.delete()
expireCursor.continue()
})
})
}
else console.warn(`IndexedDB not supported`)
async function put(url: string | null, id: string): Promise<void>
{
return await new Promise((resolve, reject) =>
{
const store = openRequest.result.transaction("store", "readwrite").objectStore("store")
if (!store) return resolve()
const request = store.put({ value: url, expireAt: new Date(Date.now() + 24 * 60 * 60 * 1000) }, id)
request.addEventListener('success', () => resolve())
request.addEventListener('error', () => reject(request.error))
})
}
async function get(id: string): Promise<string | null>
{
return (await new Promise((resolve, reject) =>
{
const store = openRequest.result.transaction("store", "readonly").objectStore("store")
if (!store) return resolve(null)
const request = store.get(id)
request.addEventListener('success', () => resolve(request.result))
request.addEventListener('error', () => reject(request.error))
}) as any)?.value
}
export const LbryURLCache = { put, get }

8
src/scripts/ytContent.tsx
View file

@ -1,7 +1,7 @@
import { ExtensionSettings, getExtensionSettingsAsync, getSourcePlatfromSettingsFromHostname, TargetPlatformName, targetPlatformSettings } from '../common/settings'
import type { UpdateContext } from '../scripts/tabOnUpdated'
import { h, JSX, render } from 'preact'
import { YtIdResolverDescriptor, ytService } from '../common/yt'
import { getExtensionSettingsAsync, getSourcePlatfromSettingsFromHostname, TargetPlatformName, targetPlatformSettings } from '../common/settings'
import { resolveById, YtIdResolverDescriptor } from '../common/yt'
import type { UpdateContext } from '../scripts/tabOnUpdated'
const sleep = (t: number) => new Promise(resolve => setTimeout(resolve, t));
@ -177,7 +177,7 @@ window.addEventListener('load', async () =>
const videoId = url.searchParams.get('v')
if (!videoId) return
const descriptor: YtIdResolverDescriptor = { id: videoId, type: 'video' }
const lbryPathname = (await ytService.resolveById([descriptor]))[0]
const lbryPathname = (await resolveById([descriptor]))[0]
if (!lbryPathname) return
updateButton({ descriptor, lbryPathname, redirect: settings.redirect, targetPlatform: settings.targetPlatform })
}

10
src/tools/YTtoLBRY.tsx
View file

@ -1,7 +1,7 @@
import { h, render } from 'preact'
import { useState } from 'preact/hooks'
import { getExtensionSettingsAsync, targetPlatformSettings } from '../common/settings'
import { getFileContent, ytService } from '../common/yt'
import { getFileContent, getSubsFromCsv, getSubsFromJson, getSubsFromOpml, resolveById } from '../common/yt'
import readme from './README.md'
@ -16,10 +16,10 @@ async function lbryChannelsFromFile(file: File) {
const ext = file.name.split('.').pop()?.toLowerCase();
const ids = new Set((
ext === 'xml' || ext == 'opml' ? ytService.readOpml :
ext === 'csv' ? ytService.readCsv :
ytService.readJson)(await getFileContent(file)))
const lbryUrls = await ytService.resolveById(
ext === 'xml' || ext == 'opml' ? getSubsFromOpml :
ext === 'csv' ? getSubsFromCsv :
getSubsFromJson)(await getFileContent(file)))
const lbryUrls = await resolveById(
Array.from(ids).map(id => ({ id, type: 'channel' } as const)),
(progress) => render(<YTtoLBRY progress={progress} />, document.getElementById('root')!));
const { targetPlatform: platform } = await getExtensionSettingsAsync();