LBRYFoundation/Watch-on-LBRY
1
0
Fork 0
mirror of https://github.com/LBRYFoundation/Watch-on-LBRY.git synced 2025-08-23 17:47:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

🥞 Refactor

Browse source
This commit is contained in:
Shiba 2022-01-09 19:10:27 +00:00
parent 75cb9cf01d
commit cb4b4f4b2e
4 changed files with 239 additions and 230 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

401
src/common/yt/index.ts
View file

@ -1,271 +1,230 @@
import chunk from 'lodash/chunk' import chunk from 'lodash/chunk'
import groupBy from 'lodash/groupBy' import groupBy from 'lodash/groupBy'
import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YtUrlResolveResponsePath, ytUrlResolversSettings } from '../settings' import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YtUrlResolveResponsePath, ytUrlResolversSettings } from '../settings'
import { LbryURLCache } from './urlCache'
// const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS // const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS
const QUERY_CHUNK_SIZE = 300; const QUERY_CHUNK_SIZE = 300
interface YtExportedJsonSubscription { interface YtExportedJsonSubscription
id: string; {
etag: string; id: string
title: string; etag: string
snippet: { title: string
description: string; snippet: {
resourceId: { description: string
channelId: string; resourceId: {
}; channelId: string
}; }
}
} }
export interface YtIdResolverDescriptor { export interface YtIdResolverDescriptor
id: string {
type: 'channel' | 'video' id: string
type: 'channel' | 'video'
} }
/** /**
* @param file to load * @param file to load
* @returns a promise with the file as a string * @returns a promise with the file as a string
*/ */
export function getFileContent(file: File): Promise<string> { export function getFileContent(file: File): Promise<string>
return new Promise((resolve, reject) => { {
const reader = new FileReader(); return new Promise((resolve, reject) =>
reader.addEventListener('load', event => resolve(event.target?.result as string || '')); {
reader.addEventListener('error', () => { const reader = new FileReader()
reader.abort(); reader.addEventListener('load', event => resolve(event.target?.result as string || ''))
reject(new DOMException(`Could not read ${file.name}`)); reader.addEventListener('error', () =>
}); {
reader.readAsText(file); reader.abort()
}); reject(new DOMException(`Could not read ${file.name}`))
})
reader.readAsText(file)
})
} }
export const ytService = (() => { /**
/** * Extracts the channelID from a YT URL.
* Extracts the channelID from a YT URL. *
* * Handles these two types of YT URLs:
* Handles these two types of YT URLs: * * /feeds/videos.xml?channel_id=*
* * /feeds/videos.xml?channel_id=* * * /channel/*
* * /channel/* */
*/ export function getChannelId(channelURL: string)
function getChannelId(channelURL: string) { {
const match = channelURL.match(/channel\/([^\s?]*)/); const match = channelURL.match(/channel\/([^\s?]*)/)
return match ? match[1] : new URL(channelURL).searchParams.get('channel_id'); return match ? match[1] : new URL(channelURL).searchParams.get('channel_id')
} }
/** /**
* Reads the array of YT channels from an OPML file * Reads the array of YT channels from an OPML file
* *
* @param opmlContents an opml file as as tring * @param opmlContents an opml file as as tring
* @returns the channel IDs * @returns the channel IDs
*/ */
function readOpml(opmlContents: string): string[] { export function getSubsFromOpml(opmlContents: string): string[]
const opml = new DOMParser().parseFromString(opmlContents, 'application/xml'); {
const opml = new DOMParser().parseFromString(opmlContents, 'application/xml')
opmlContents = '' opmlContents = ''
return Array.from(opml.querySelectorAll('outline > outline')) return Array.from(opml.querySelectorAll('outline > outline'))
.map(outline => outline.getAttribute('xmlUrl')) .map(outline => outline.getAttribute('xmlUrl'))
.filter((url): url is string => !!url) .filter((url): url is string => !!url)
.map(url => getChannelId(url)) .map(url => getChannelId(url))
.filter((url): url is string => !!url); // we don't want it if it's empty .filter((url): url is string => !!url) // we don't want it if it's empty
} }
/** /**
* Reads an array of YT channel IDs from the YT subscriptions JSON file * Reads an array of YT channel IDs from the YT subscriptions JSON file
* *
* @param jsonContents a JSON file as a string * @param jsonContents a JSON file as a string
* @returns the channel IDs * @returns the channel IDs
*/ */
function readJson(jsonContents: string): string[] { export function getSubsFromJson(jsonContents: string): string[]
const subscriptions: YtExportedJsonSubscription[] = JSON.parse(jsonContents); {
const subscriptions: YtExportedJsonSubscription[] = JSON.parse(jsonContents)
jsonContents = '' jsonContents = ''
return subscriptions.map(sub => sub.snippet.resourceId.channelId); return subscriptions.map(sub => sub.snippet.resourceId.channelId)
} }
/** /**
* Reads an array of YT channel IDs from the YT subscriptions CSV file * Reads an array of YT channel IDs from the YT subscriptions CSV file
* *
* @param csvContent a CSV file as a string * @param csvContent a CSV file as a string
* @returns the channel IDs * @returns the channel IDs
*/ */
function readCsv(csvContent: string): string[] { export function getSubsFromCsv(csvContent: string): string[]
{
const rows = csvContent.split('\n') const rows = csvContent.split('\n')
csvContent = '' csvContent = ''
return rows.slice(1).map((row) => row.substring(0, row.indexOf(','))) return rows.slice(1).map((row) => row.substring(0, row.indexOf(',')))
} }
const URLResolverCache = (() => /**
{ * @param descriptorsWithIndex YT resource IDs to check
const openRequest = self.indexedDB?.open("yt-url-resolver-cache") * @returns a promise with the list of channels that were found on lbry
*/
if (openRequest) export async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]>
{ {
openRequest.addEventListener('upgradeneeded', () => openRequest.result.createObjectStore("store").createIndex("expireAt", "expireAt")) const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({ ...descriptor, index }))
// Delete Expired
openRequest.addEventListener('success', () =>
{
const transaction = openRequest.result.transaction("store", "readwrite")
const range = IDBKeyRange.upperBound(new Date())
const expireAtCursorRequest = transaction.objectStore("store").index("expireAt").openCursor(range)
expireAtCursorRequest.addEventListener('success', () =>
{
const expireCursor = expireAtCursorRequest.result
if (!expireCursor) return
expireCursor.delete()
expireCursor.continue()
})
})
}
else console.warn(`IndexedDB not supported`)
async function put(url: string | null, id: string): Promise<void>
{
return await new Promise((resolve, reject) =>
{
const store = openRequest.result.transaction("store", "readwrite").objectStore("store")
if (!store) return resolve()
const request = store.put({ value: url, expireAt: new Date(Date.now() + 24 * 60 * 60 * 1000) }, id)
request.addEventListener('success', () => resolve())
request.addEventListener('error', () => reject(request.error))
})
}
async function get(id: string): Promise<string | null>
{
return (await new Promise((resolve, reject) =>
{
const store = openRequest.result.transaction("store", "readonly").objectStore("store")
if (!store) return resolve(null)
const request = store.get(id)
request.addEventListener('success', () => resolve(request.result))
request.addEventListener('error', () => reject(request.error))
}) as any)?.value
}
return { put, get }
})()
/**
* @param descriptorsWithIndex YT resource IDs to check
* @returns a promise with the list of channels that were found on lbry
*/
async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]> {
const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({...descriptor, index}))
descriptors = null as any descriptors = null as any
const results: (string | null)[] = [] const results: (string | null)[] = []
await Promise.all(descriptorsWithIndex.map(async (descriptor, index) => { await Promise.all(descriptorsWithIndex.map(async (descriptor, index) =>
if (!descriptor) return {
const cache = await URLResolverCache.get(descriptor.id) if (!descriptor) return
const cache = await LbryURLCache.get(descriptor.id)
// Cache can be null, if there is no lbry url yet // Cache can be null, if there is no lbry url yet
if (cache !== undefined) { if (cache !== undefined)
// Directly setting it to results {
results[index] = cache // Directly setting it to results
results[index] = cache
// We remove it so we dont ask it to API // We remove it so we dont ask it to API
descriptorsWithIndex.splice(index, 1) descriptorsWithIndex.splice(index, 1)
} }
})) }))
const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE); const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE)
let progressCount = 0; let progressCount = 0
await Promise.all(descriptorsChunks.map(async (descriptorChunk) => await Promise.all(descriptorsChunks.map(async (descriptorChunk) =>
{ {
const descriptorsGroupedByType: Record<YtIdResolverDescriptor['type'], typeof descriptorsWithIndex | null> = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any; const descriptorsGroupedByType: Record<YtIdResolverDescriptor['type'], typeof descriptorsWithIndex | null> = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any
const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync() const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync()
const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName] const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName]
const url = new URL(`https://${urlResolverSetting.hostname}`); const url = new URL(`https://${urlResolverSetting.hostname}`)
function followResponsePath<T>(response: any, responsePath: YtUrlResolveResponsePath) function followResponsePath<T>(response: any, responsePath: YtUrlResolveResponsePath)
{
for (const path of responsePath)
{ {
switch (typeof path) for (const path of responsePath)
{
case 'string':
case 'number':
response = response[path]
break
default:
switch (path)
{
case Keys:
response = Object.keys(response)
break
case Values:
response = Object.values(response)
break
}
}
}
return response as T
}
async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsWithIndex)
{
url.pathname = urlResolverFunction.pathname
if (urlResolverFunction.paramArraySeperator === SingleValueAtATime)
{
await Promise.all(descriptorsGroup.map(async (descriptor) => {
switch (null)
{ {
default: switch (typeof path)
if (!descriptor.id) break {
url.searchParams.set(urlResolverFunction.paramName, descriptor.id) case 'string':
case 'number':
const apiResponse = await fetch(url.toString(), { cache: 'no-store' }); response = response[path]
if (!apiResponse.ok) { break
// Some API might not respond with 200 if it can't find the url default:
if (apiResponse.status === 404) await URLResolverCache.put(null, descriptor.id) switch (path)
break {
} case Keys:
response = Object.keys(response)
const value = followResponsePath<string>(await apiResponse.json(), urlResolverFunction.responsePath) break
if (value) results[descriptor.index] = value case Values:
await URLResolverCache.put(value, descriptor.id) response = Object.values(response)
break
}
}
} }
progressCount++ return response as T
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
}))
} }
else
async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsWithIndex)
{ {
url.pathname = urlResolverFunction.pathname
switch (null) if (urlResolverFunction.paramArraySeperator === SingleValueAtATime)
{ {
default: await Promise.all(descriptorsGroup.map(async (descriptor) =>
url.searchParams {
.set(urlResolverFunction.paramName, descriptorsGroup switch (null)
.map((descriptor) => descriptor.id) {
.filter((descriptorId) => descriptorId) default:
.join(urlResolverFunction.paramArraySeperator) if (!descriptor.id) break
) url.searchParams.set(urlResolverFunction.paramName, descriptor.id)
const apiResponse = await fetch(url.toString(), { cache: 'no-store' }); const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
if (!apiResponse.ok) break if (!apiResponse.ok)
const values = followResponsePath<string[]>(await apiResponse.json(), urlResolverFunction.responsePath) {
// Some API might not respond with 200 if it can't find the url
if (apiResponse.status === 404) await LbryURLCache.put(null, descriptor.id)
break
}
await Promise.all(values.map(async (value, index) => { const value = followResponsePath<string>(await apiResponse.json(), urlResolverFunction.responsePath)
const descriptor = descriptorsGroup[index] if (value) results[descriptor.index] = value
if (value) results[descriptor.index] = value await LbryURLCache.put(value, descriptor.id)
await URLResolverCache.put(value, descriptor.id) }
})) progressCount++
} if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
progressCount += descriptorsGroup.length }))
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length) }
else
{
switch (null)
{
default:
url.searchParams
.set(urlResolverFunction.paramName, descriptorsGroup
.map((descriptor) => descriptor.id)
.filter((descriptorId) => descriptorId)
.join(urlResolverFunction.paramArraySeperator)
)
const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
if (!apiResponse.ok) break
const values = followResponsePath<string[]>(await apiResponse.json(), urlResolverFunction.responsePath)
await Promise.all(values.map(async (value, index) =>
{
const descriptor = descriptorsGroup[index]
if (value) results[descriptor.index] = value
await LbryURLCache.put(value, descriptor.id)
}))
}
progressCount += descriptorsGroup.length
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
}
} }
}
if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel']) if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel'])
if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video']) if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video'])
}))
}));
return results return results
} }
return { readCsv, readJson, readOpml, resolveById }
})()

50
src/common/yt/urlCache.ts Normal file
View file

@ -0,0 +1,50 @@
const openRequest = self.indexedDB?.open("yt-url-resolver-cache")
if (openRequest)
{
openRequest.addEventListener('upgradeneeded', () => openRequest.result.createObjectStore("store").createIndex("expireAt", "expireAt"))
// Delete Expired
openRequest.addEventListener('success', () =>
{
const transaction = openRequest.result.transaction("store", "readwrite")
const range = IDBKeyRange.upperBound(new Date())
const expireAtCursorRequest = transaction.objectStore("store").index("expireAt").openCursor(range)
expireAtCursorRequest.addEventListener('success', () =>
{
const expireCursor = expireAtCursorRequest.result
if (!expireCursor) return
expireCursor.delete()
expireCursor.continue()
})
})
}
else console.warn(`IndexedDB not supported`)
async function put(url: string | null, id: string): Promise<void>
{
return await new Promise((resolve, reject) =>
{
const store = openRequest.result.transaction("store", "readwrite").objectStore("store")
if (!store) return resolve()
const request = store.put({ value: url, expireAt: new Date(Date.now() + 24 * 60 * 60 * 1000) }, id)
request.addEventListener('success', () => resolve())
request.addEventListener('error', () => reject(request.error))
})
}
async function get(id: string): Promise<string | null>
{
return (await new Promise((resolve, reject) =>
{
const store = openRequest.result.transaction("store", "readonly").objectStore("store")
if (!store) return resolve(null)
const request = store.get(id)
request.addEventListener('success', () => resolve(request.result))
request.addEventListener('error', () => reject(request.error))
}) as any)?.value
}
export const LbryURLCache = { put, get }

8
src/scripts/ytContent.tsx
View file

@ -1,7 +1,7 @@
import { ExtensionSettings, getExtensionSettingsAsync, getSourcePlatfromSettingsFromHostname, TargetPlatformName, targetPlatformSettings } from '../common/settings'
import type { UpdateContext } from '../scripts/tabOnUpdated'
import { h, JSX, render } from 'preact' import { h, JSX, render } from 'preact'
import { YtIdResolverDescriptor, ytService } from '../common/yt' import { getExtensionSettingsAsync, getSourcePlatfromSettingsFromHostname, TargetPlatformName, targetPlatformSettings } from '../common/settings'
import { resolveById, YtIdResolverDescriptor } from '../common/yt'
import type { UpdateContext } from '../scripts/tabOnUpdated'
const sleep = (t: number) => new Promise(resolve => setTimeout(resolve, t)); const sleep = (t: number) => new Promise(resolve => setTimeout(resolve, t));
@ -177,7 +177,7 @@ window.addEventListener('load', async () =>
const videoId = url.searchParams.get('v') const videoId = url.searchParams.get('v')
if (!videoId) return if (!videoId) return
const descriptor: YtIdResolverDescriptor = { id: videoId, type: 'video' } const descriptor: YtIdResolverDescriptor = { id: videoId, type: 'video' }
const lbryPathname = (await ytService.resolveById([descriptor]))[0] const lbryPathname = (await resolveById([descriptor]))[0]
if (!lbryPathname) return if (!lbryPathname) return
updateButton({ descriptor, lbryPathname, redirect: settings.redirect, targetPlatform: settings.targetPlatform }) updateButton({ descriptor, lbryPathname, redirect: settings.redirect, targetPlatform: settings.targetPlatform })
} }

10
src/tools/YTtoLBRY.tsx
View file

@ -1,7 +1,7 @@
import { h, render } from 'preact' import { h, render } from 'preact'
import { useState } from 'preact/hooks' import { useState } from 'preact/hooks'
import { getExtensionSettingsAsync, targetPlatformSettings } from '../common/settings' import { getExtensionSettingsAsync, targetPlatformSettings } from '../common/settings'
import { getFileContent, ytService } from '../common/yt' import { getFileContent, getSubsFromCsv, getSubsFromJson, getSubsFromOpml, resolveById } from '../common/yt'
import readme from './README.md' import readme from './README.md'
@ -16,10 +16,10 @@ async function lbryChannelsFromFile(file: File) {
const ext = file.name.split('.').pop()?.toLowerCase(); const ext = file.name.split('.').pop()?.toLowerCase();
const ids = new Set(( const ids = new Set((
ext === 'xml' || ext == 'opml' ? ytService.readOpml : ext === 'xml' || ext == 'opml' ? getSubsFromOpml :
ext === 'csv' ? ytService.readCsv : ext === 'csv' ? getSubsFromCsv :
ytService.readJson)(await getFileContent(file))) getSubsFromJson)(await getFileContent(file)))
const lbryUrls = await ytService.resolveById( const lbryUrls = await resolveById(
Array.from(ids).map(id => ({ id, type: 'channel' } as const)), Array.from(ids).map(id => ({ id, type: 'channel' } as const)),
(progress) => render(<YTtoLBRY progress={progress} />, document.getElementById('root')!)); (progress) => render(<YTtoLBRY progress={progress} />, document.getElementById('root')!));
const { targetPlatform: platform } = await getExtensionSettingsAsync(); const { targetPlatform: platform } = await getExtensionSettingsAsync();