mirror of
https://github.com/LBRYFoundation/Watch-on-LBRY.git
synced 2025-08-23 17:47:26 +00:00
🥡 Refactor
This commit is contained in:
Shiba
parent
4f8e807a65
commit
2b91436900
4 changed files with 188 additions and 187 deletions
|
|
@ -1,10 +1,3 @@
|
||||||
import chunk from 'lodash/chunk'
|
|
||||||
import groupBy from 'lodash/groupBy'
|
|
||||||
import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YtUrlResolveResponsePath, ytUrlResolversSettings } from '../settings'
|
|
||||||
import { LbryPathnameCache } from './urlCache'
|
|
||||||
|
|
||||||
// const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS
|
|
||||||
const QUERY_CHUNK_SIZE = 300
|
|
||||||
|
|
||||||
interface YtExportedJsonSubscription
|
interface YtExportedJsonSubscription
|
||||||
{
|
{
|
||||||
|
|
@ -19,11 +12,6 @@ interface YtExportedJsonSubscription
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface YtIdResolverDescriptor
|
|
||||||
{
|
|
||||||
id: string
|
|
||||||
type: 'channel' | 'video'
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param file to load
|
* @param file to load
|
||||||
|
|
@ -44,6 +32,50 @@ export function getFileContent(file: File): Promise<string>
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads the array of YT channels from an OPML file
|
||||||
|
*
|
||||||
|
* @param opmlContents an opml file as as tring
|
||||||
|
* @returns the channel IDs
|
||||||
|
*/
|
||||||
|
export function getSubsFromOpml(opmlContents: string): string[]
|
||||||
|
{
|
||||||
|
const opml = new DOMParser().parseFromString(opmlContents, 'application/xml')
|
||||||
|
opmlContents = ''
|
||||||
|
return Array.from(opml.querySelectorAll('outline > outline'))
|
||||||
|
.map(outline => outline.getAttribute('xmlUrl'))
|
||||||
|
.filter((url): url is string => !!url)
|
||||||
|
.map(url => getChannelId(url))
|
||||||
|
.filter((url): url is string => !!url) // we don't want it if it's empty
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads an array of YT channel IDs from the YT subscriptions JSON file
|
||||||
|
*
|
||||||
|
* @param jsonContents a JSON file as a string
|
||||||
|
* @returns the channel IDs
|
||||||
|
*/
|
||||||
|
export function getSubsFromJson(jsonContents: string): string[]
|
||||||
|
{
|
||||||
|
const subscriptions: YtExportedJsonSubscription[] = JSON.parse(jsonContents)
|
||||||
|
jsonContents = ''
|
||||||
|
return subscriptions.map(sub => sub.snippet.resourceId.channelId)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads an array of YT channel IDs from the YT subscriptions CSV file
|
||||||
|
*
|
||||||
|
* @param csvContent a CSV file as a string
|
||||||
|
* @returns the channel IDs
|
||||||
|
*/
|
||||||
|
export function getSubsFromCsv(csvContent: string): string[]
|
||||||
|
{
|
||||||
|
const rows = csvContent.split('\n')
|
||||||
|
csvContent = ''
|
||||||
|
return rows.slice(1).map((row) => row.substring(0, row.indexOf(',')))
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Extracts the channelID from a YT URL.
|
* Extracts the channelID from a YT URL.
|
||||||
*
|
*
|
||||||
|
|
@ -77,177 +109,4 @@ export function parseYouTubeURLTimeString(timeString: string)
|
||||||
total += t
|
total += t
|
||||||
}
|
}
|
||||||
return total.toString()
|
return total.toString()
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reads the array of YT channels from an OPML file
|
|
||||||
*
|
|
||||||
* @param opmlContents an opml file as as tring
|
|
||||||
* @returns the channel IDs
|
|
||||||
*/
|
|
||||||
export function getSubsFromOpml(opmlContents: string): string[]
|
|
||||||
{
|
|
||||||
const opml = new DOMParser().parseFromString(opmlContents, 'application/xml')
|
|
||||||
opmlContents = ''
|
|
||||||
return Array.from(opml.querySelectorAll('outline > outline'))
|
|
||||||
.map(outline => outline.getAttribute('xmlUrl'))
|
|
||||||
.filter((url): url is string => !!url)
|
|
||||||
.map(url => getChannelId(url))
|
|
||||||
.filter((url): url is string => !!url) // we don't want it if it's empty
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reads an array of YT channel IDs from the YT subscriptions JSON file
|
|
||||||
*
|
|
||||||
* @param jsonContents a JSON file as a string
|
|
||||||
* @returns the channel IDs
|
|
||||||
*/
|
|
||||||
export function getSubsFromJson(jsonContents: string): string[]
|
|
||||||
{
|
|
||||||
const subscriptions: YtExportedJsonSubscription[] = JSON.parse(jsonContents)
|
|
||||||
jsonContents = ''
|
|
||||||
return subscriptions.map(sub => sub.snippet.resourceId.channelId)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reads an array of YT channel IDs from the YT subscriptions CSV file
|
|
||||||
*
|
|
||||||
* @param csvContent a CSV file as a string
|
|
||||||
* @returns the channel IDs
|
|
||||||
*/
|
|
||||||
export function getSubsFromCsv(csvContent: string): string[]
|
|
||||||
{
|
|
||||||
const rows = csvContent.split('\n')
|
|
||||||
csvContent = ''
|
|
||||||
return rows.slice(1).map((row) => row.substring(0, row.indexOf(',')))
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param descriptorsWithIndex YT resource IDs to check
|
|
||||||
* @returns a promise with the list of channels that were found on lbry
|
|
||||||
*/
|
|
||||||
export async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]>
|
|
||||||
{
|
|
||||||
const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({ ...descriptor, index }))
|
|
||||||
descriptors = null as any
|
|
||||||
const results: (string | null)[] = []
|
|
||||||
|
|
||||||
await Promise.all(descriptorsWithIndex.map(async (descriptor, index) =>
|
|
||||||
{
|
|
||||||
if (!descriptor) return
|
|
||||||
const cache = await LbryPathnameCache.get(descriptor.id)
|
|
||||||
|
|
||||||
// Cache can be null, if there is no lbry url yet
|
|
||||||
if (cache !== undefined)
|
|
||||||
{
|
|
||||||
// Directly setting it to results
|
|
||||||
results[index] = cache
|
|
||||||
|
|
||||||
// We remove it so we dont ask it to API
|
|
||||||
descriptorsWithIndex.splice(index, 1)
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
|
|
||||||
const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE)
|
|
||||||
let progressCount = 0
|
|
||||||
await Promise.all(descriptorsChunks.map(async (descriptorChunk) =>
|
|
||||||
{
|
|
||||||
const descriptorsGroupedByType: Record<YtIdResolverDescriptor['type'], typeof descriptorsWithIndex | null> = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any
|
|
||||||
|
|
||||||
const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync()
|
|
||||||
const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName]
|
|
||||||
|
|
||||||
const url = new URL(`https://${urlResolverSetting.hostname}`)
|
|
||||||
|
|
||||||
function followResponsePath<T>(response: any, responsePath: YtUrlResolveResponsePath)
|
|
||||||
{
|
|
||||||
for (const path of responsePath)
|
|
||||||
{
|
|
||||||
switch (typeof path)
|
|
||||||
{
|
|
||||||
case 'string':
|
|
||||||
case 'number':
|
|
||||||
response = response[path]
|
|
||||||
break
|
|
||||||
default:
|
|
||||||
switch (path)
|
|
||||||
{
|
|
||||||
case Keys:
|
|
||||||
response = Object.keys(response)
|
|
||||||
break
|
|
||||||
case Values:
|
|
||||||
response = Object.values(response)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return response as T
|
|
||||||
}
|
|
||||||
|
|
||||||
async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsWithIndex)
|
|
||||||
{
|
|
||||||
url.pathname = urlResolverFunction.pathname
|
|
||||||
|
|
||||||
if (urlResolverFunction.paramArraySeperator === SingleValueAtATime)
|
|
||||||
{
|
|
||||||
await Promise.all(descriptorsGroup.map(async (descriptor) =>
|
|
||||||
{
|
|
||||||
switch (null)
|
|
||||||
{
|
|
||||||
default:
|
|
||||||
if (!descriptor.id) break
|
|
||||||
url.searchParams.set(urlResolverFunction.paramName, descriptor.id)
|
|
||||||
|
|
||||||
const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
|
|
||||||
if (!apiResponse.ok)
|
|
||||||
{
|
|
||||||
// Some API might not respond with 200 if it can't find the url
|
|
||||||
if (apiResponse.status === 404) await LbryPathnameCache.put(null, descriptor.id)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
const value = followResponsePath<string>(await apiResponse.json(), urlResolverFunction.responsePath)
|
|
||||||
if (value) results[descriptor.index] = value
|
|
||||||
await LbryPathnameCache.put(value, descriptor.id)
|
|
||||||
}
|
|
||||||
progressCount++
|
|
||||||
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
|
|
||||||
switch (null)
|
|
||||||
{
|
|
||||||
default:
|
|
||||||
url.searchParams
|
|
||||||
.set(urlResolverFunction.paramName, descriptorsGroup
|
|
||||||
.map((descriptor) => descriptor.id)
|
|
||||||
.filter((descriptorId) => descriptorId)
|
|
||||||
.join(urlResolverFunction.paramArraySeperator)
|
|
||||||
)
|
|
||||||
|
|
||||||
const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
|
|
||||||
if (!apiResponse.ok) break
|
|
||||||
const values = followResponsePath<string[]>(await apiResponse.json(), urlResolverFunction.responsePath)
|
|
||||||
|
|
||||||
await Promise.all(values.map(async (value, index) =>
|
|
||||||
{
|
|
||||||
const descriptor = descriptorsGroup[index]
|
|
||||||
if (value) results[descriptor.index] = value
|
|
||||||
await LbryPathnameCache.put(value, descriptor.id)
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
progressCount += descriptorsGroup.length
|
|
||||||
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel'])
|
|
||||||
if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video'])
|
|
||||||
|
|
||||||
}))
|
|
||||||
|
|
||||||
return results
|
|
||||||
}
|
}
|
||||||
141
src/common/yt/urlResolve.ts
Normal file
141
src/common/yt/urlResolve.ts
Normal file
|
|
@ -0,0 +1,141 @@
|
||||||
|
import { chunk, groupBy } from "lodash"
|
||||||
|
import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YtUrlResolveResponsePath, ytUrlResolversSettings } from "../settings"
|
||||||
|
import { LbryPathnameCache } from "./urlCache"
|
||||||
|
|
||||||
|
// const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS
|
||||||
|
const QUERY_CHUNK_SIZE = 300
|
||||||
|
|
||||||
|
export interface YtIdResolverDescriptor
|
||||||
|
{
|
||||||
|
id: string
|
||||||
|
type: 'channel' | 'video'
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param descriptorsWithIndex YT resource IDs to check
|
||||||
|
* @returns a promise with the list of channels that were found on lbry
|
||||||
|
*/
|
||||||
|
export async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]>
|
||||||
|
{
|
||||||
|
const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({ ...descriptor, index }))
|
||||||
|
descriptors = null as any
|
||||||
|
const results: (string | null)[] = []
|
||||||
|
|
||||||
|
await Promise.all(descriptorsWithIndex.map(async (descriptor, index) =>
|
||||||
|
{
|
||||||
|
if (!descriptor) return
|
||||||
|
const cache = await LbryPathnameCache.get(descriptor.id)
|
||||||
|
|
||||||
|
// Cache can be null, if there is no lbry url yet
|
||||||
|
if (cache !== undefined)
|
||||||
|
{
|
||||||
|
// Directly setting it to results
|
||||||
|
results[index] = cache
|
||||||
|
|
||||||
|
// We remove it so we dont ask it to API
|
||||||
|
descriptorsWithIndex.splice(index, 1)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
|
||||||
|
const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE)
|
||||||
|
let progressCount = 0
|
||||||
|
await Promise.all(descriptorsChunks.map(async (descriptorChunk) =>
|
||||||
|
{
|
||||||
|
const descriptorsGroupedByType: Record<YtIdResolverDescriptor['type'], typeof descriptorsWithIndex | null> = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any
|
||||||
|
|
||||||
|
const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync()
|
||||||
|
const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName]
|
||||||
|
|
||||||
|
const url = new URL(`https://${urlResolverSetting.hostname}`)
|
||||||
|
|
||||||
|
function followResponsePath<T>(response: any, responsePath: YtUrlResolveResponsePath)
|
||||||
|
{
|
||||||
|
for (const path of responsePath)
|
||||||
|
{
|
||||||
|
switch (typeof path)
|
||||||
|
{
|
||||||
|
case 'string':
|
||||||
|
case 'number':
|
||||||
|
response = response[path]
|
||||||
|
break
|
||||||
|
default:
|
||||||
|
switch (path)
|
||||||
|
{
|
||||||
|
case Keys:
|
||||||
|
response = Object.keys(response)
|
||||||
|
break
|
||||||
|
case Values:
|
||||||
|
response = Object.values(response)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return response as T
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsWithIndex)
|
||||||
|
{
|
||||||
|
url.pathname = urlResolverFunction.pathname
|
||||||
|
|
||||||
|
if (urlResolverFunction.paramArraySeperator === SingleValueAtATime)
|
||||||
|
{
|
||||||
|
await Promise.all(descriptorsGroup.map(async (descriptor) =>
|
||||||
|
{
|
||||||
|
switch (null)
|
||||||
|
{
|
||||||
|
default:
|
||||||
|
if (!descriptor.id) break
|
||||||
|
url.searchParams.set(urlResolverFunction.paramName, descriptor.id)
|
||||||
|
|
||||||
|
const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
|
||||||
|
if (!apiResponse.ok)
|
||||||
|
{
|
||||||
|
// Some API might not respond with 200 if it can't find the url
|
||||||
|
if (apiResponse.status === 404) await LbryPathnameCache.put(null, descriptor.id)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
const value = followResponsePath<string>(await apiResponse.json(), urlResolverFunction.responsePath)
|
||||||
|
if (value) results[descriptor.index] = value
|
||||||
|
await LbryPathnameCache.put(value, descriptor.id)
|
||||||
|
}
|
||||||
|
progressCount++
|
||||||
|
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
|
||||||
|
switch (null)
|
||||||
|
{
|
||||||
|
default:
|
||||||
|
url.searchParams
|
||||||
|
.set(urlResolverFunction.paramName, descriptorsGroup
|
||||||
|
.map((descriptor) => descriptor.id)
|
||||||
|
.filter((descriptorId) => descriptorId)
|
||||||
|
.join(urlResolverFunction.paramArraySeperator)
|
||||||
|
)
|
||||||
|
|
||||||
|
const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
|
||||||
|
if (!apiResponse.ok) break
|
||||||
|
const values = followResponsePath<string[]>(await apiResponse.json(), urlResolverFunction.responsePath)
|
||||||
|
|
||||||
|
await Promise.all(values.map(async (value, index) =>
|
||||||
|
{
|
||||||
|
const descriptor = descriptorsGroup[index]
|
||||||
|
if (value) results[descriptor.index] = value
|
||||||
|
await LbryPathnameCache.put(value, descriptor.id)
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
progressCount += descriptorsGroup.length
|
||||||
|
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel'])
|
||||||
|
if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video'])
|
||||||
|
|
||||||
|
}))
|
||||||
|
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import { parseProtocolUrl } from '../common/lbry-url'
|
import { parseProtocolUrl } from '../common/lbry-url'
|
||||||
import { resolveById, YtIdResolverDescriptor } from '../common/yt'
|
import { resolveById, YtIdResolverDescriptor } from '../common/yt/urlResolve'
|
||||||
async function resolveYT(descriptor: YtIdResolverDescriptor) {
|
async function resolveYT(descriptor: YtIdResolverDescriptor) {
|
||||||
const lbryProtocolUrl: string | null = await resolveById([descriptor]).then(a => a[0]);
|
const lbryProtocolUrl: string | null = await resolveById([descriptor]).then(a => a[0]);
|
||||||
const segments = parseProtocolUrl(lbryProtocolUrl || '', { encode: true });
|
const segments = parseProtocolUrl(lbryProtocolUrl || '', { encode: true });
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,8 @@
|
||||||
import { h, render } from 'preact'
|
import { h, render } from 'preact'
|
||||||
import { useState } from 'preact/hooks'
|
import { useState } from 'preact/hooks'
|
||||||
import { getExtensionSettingsAsync, targetPlatformSettings } from '../common/settings'
|
import { getExtensionSettingsAsync, targetPlatformSettings } from '../common/settings'
|
||||||
import { getFileContent, getSubsFromCsv, getSubsFromJson, getSubsFromOpml, resolveById } from '../common/yt'
|
import { getFileContent, getSubsFromCsv, getSubsFromJson, getSubsFromOpml } from '../common/yt'
|
||||||
|
import { resolveById } from '../common/yt/urlResolve'
|
||||||
import readme from './README.md'
|
import readme from './README.md'
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue