mirror of
https://github.com/LBRYFoundation/Watch-on-LBRY.git
synced 2025-08-23 17:47:26 +00:00
🥡 Refactor
This commit is contained in:
Shiba
parent
4f8e807a65
commit
2b91436900
4 changed files with 188 additions and 187 deletions
|
|
@ -1,10 +1,3 @@
|
|||
import chunk from 'lodash/chunk'
|
||||
import groupBy from 'lodash/groupBy'
|
||||
import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YtUrlResolveResponsePath, ytUrlResolversSettings } from '../settings'
|
||||
import { LbryPathnameCache } from './urlCache'
|
||||
|
||||
// const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS
|
||||
const QUERY_CHUNK_SIZE = 300
|
||||
|
||||
interface YtExportedJsonSubscription
|
||||
{
|
||||
|
|
@ -19,11 +12,6 @@ interface YtExportedJsonSubscription
|
|||
}
|
||||
}
|
||||
|
||||
export interface YtIdResolverDescriptor
|
||||
{
|
||||
id: string
|
||||
type: 'channel' | 'video'
|
||||
}
|
||||
|
||||
/**
|
||||
* @param file to load
|
||||
|
|
@ -44,6 +32,50 @@ export function getFileContent(file: File): Promise<string>
|
|||
})
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Reads the array of YT channels from an OPML file
|
||||
*
|
||||
* @param opmlContents an opml file as as tring
|
||||
* @returns the channel IDs
|
||||
*/
|
||||
export function getSubsFromOpml(opmlContents: string): string[]
|
||||
{
|
||||
const opml = new DOMParser().parseFromString(opmlContents, 'application/xml')
|
||||
opmlContents = ''
|
||||
return Array.from(opml.querySelectorAll('outline > outline'))
|
||||
.map(outline => outline.getAttribute('xmlUrl'))
|
||||
.filter((url): url is string => !!url)
|
||||
.map(url => getChannelId(url))
|
||||
.filter((url): url is string => !!url) // we don't want it if it's empty
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads an array of YT channel IDs from the YT subscriptions JSON file
|
||||
*
|
||||
* @param jsonContents a JSON file as a string
|
||||
* @returns the channel IDs
|
||||
*/
|
||||
export function getSubsFromJson(jsonContents: string): string[]
|
||||
{
|
||||
const subscriptions: YtExportedJsonSubscription[] = JSON.parse(jsonContents)
|
||||
jsonContents = ''
|
||||
return subscriptions.map(sub => sub.snippet.resourceId.channelId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads an array of YT channel IDs from the YT subscriptions CSV file
|
||||
*
|
||||
* @param csvContent a CSV file as a string
|
||||
* @returns the channel IDs
|
||||
*/
|
||||
export function getSubsFromCsv(csvContent: string): string[]
|
||||
{
|
||||
const rows = csvContent.split('\n')
|
||||
csvContent = ''
|
||||
return rows.slice(1).map((row) => row.substring(0, row.indexOf(',')))
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the channelID from a YT URL.
|
||||
*
|
||||
|
|
@ -77,177 +109,4 @@ export function parseYouTubeURLTimeString(timeString: string)
|
|||
total += t
|
||||
}
|
||||
return total.toString()
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Reads the array of YT channels from an OPML file
|
||||
*
|
||||
* @param opmlContents an opml file as as tring
|
||||
* @returns the channel IDs
|
||||
*/
|
||||
export function getSubsFromOpml(opmlContents: string): string[]
|
||||
{
|
||||
const opml = new DOMParser().parseFromString(opmlContents, 'application/xml')
|
||||
opmlContents = ''
|
||||
return Array.from(opml.querySelectorAll('outline > outline'))
|
||||
.map(outline => outline.getAttribute('xmlUrl'))
|
||||
.filter((url): url is string => !!url)
|
||||
.map(url => getChannelId(url))
|
||||
.filter((url): url is string => !!url) // we don't want it if it's empty
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads an array of YT channel IDs from the YT subscriptions JSON file
|
||||
*
|
||||
* @param jsonContents a JSON file as a string
|
||||
* @returns the channel IDs
|
||||
*/
|
||||
export function getSubsFromJson(jsonContents: string): string[]
|
||||
{
|
||||
const subscriptions: YtExportedJsonSubscription[] = JSON.parse(jsonContents)
|
||||
jsonContents = ''
|
||||
return subscriptions.map(sub => sub.snippet.resourceId.channelId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads an array of YT channel IDs from the YT subscriptions CSV file
|
||||
*
|
||||
* @param csvContent a CSV file as a string
|
||||
* @returns the channel IDs
|
||||
*/
|
||||
export function getSubsFromCsv(csvContent: string): string[]
|
||||
{
|
||||
const rows = csvContent.split('\n')
|
||||
csvContent = ''
|
||||
return rows.slice(1).map((row) => row.substring(0, row.indexOf(',')))
|
||||
}
|
||||
|
||||
/**
|
||||
* @param descriptorsWithIndex YT resource IDs to check
|
||||
* @returns a promise with the list of channels that were found on lbry
|
||||
*/
|
||||
export async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]>
|
||||
{
|
||||
const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({ ...descriptor, index }))
|
||||
descriptors = null as any
|
||||
const results: (string | null)[] = []
|
||||
|
||||
await Promise.all(descriptorsWithIndex.map(async (descriptor, index) =>
|
||||
{
|
||||
if (!descriptor) return
|
||||
const cache = await LbryPathnameCache.get(descriptor.id)
|
||||
|
||||
// Cache can be null, if there is no lbry url yet
|
||||
if (cache !== undefined)
|
||||
{
|
||||
// Directly setting it to results
|
||||
results[index] = cache
|
||||
|
||||
// We remove it so we dont ask it to API
|
||||
descriptorsWithIndex.splice(index, 1)
|
||||
}
|
||||
}))
|
||||
|
||||
const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE)
|
||||
let progressCount = 0
|
||||
await Promise.all(descriptorsChunks.map(async (descriptorChunk) =>
|
||||
{
|
||||
const descriptorsGroupedByType: Record<YtIdResolverDescriptor['type'], typeof descriptorsWithIndex | null> = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any
|
||||
|
||||
const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync()
|
||||
const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName]
|
||||
|
||||
const url = new URL(`https://${urlResolverSetting.hostname}`)
|
||||
|
||||
function followResponsePath<T>(response: any, responsePath: YtUrlResolveResponsePath)
|
||||
{
|
||||
for (const path of responsePath)
|
||||
{
|
||||
switch (typeof path)
|
||||
{
|
||||
case 'string':
|
||||
case 'number':
|
||||
response = response[path]
|
||||
break
|
||||
default:
|
||||
switch (path)
|
||||
{
|
||||
case Keys:
|
||||
response = Object.keys(response)
|
||||
break
|
||||
case Values:
|
||||
response = Object.values(response)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return response as T
|
||||
}
|
||||
|
||||
async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsWithIndex)
|
||||
{
|
||||
url.pathname = urlResolverFunction.pathname
|
||||
|
||||
if (urlResolverFunction.paramArraySeperator === SingleValueAtATime)
|
||||
{
|
||||
await Promise.all(descriptorsGroup.map(async (descriptor) =>
|
||||
{
|
||||
switch (null)
|
||||
{
|
||||
default:
|
||||
if (!descriptor.id) break
|
||||
url.searchParams.set(urlResolverFunction.paramName, descriptor.id)
|
||||
|
||||
const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
|
||||
if (!apiResponse.ok)
|
||||
{
|
||||
// Some API might not respond with 200 if it can't find the url
|
||||
if (apiResponse.status === 404) await LbryPathnameCache.put(null, descriptor.id)
|
||||
break
|
||||
}
|
||||
|
||||
const value = followResponsePath<string>(await apiResponse.json(), urlResolverFunction.responsePath)
|
||||
if (value) results[descriptor.index] = value
|
||||
await LbryPathnameCache.put(value, descriptor.id)
|
||||
}
|
||||
progressCount++
|
||||
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
|
||||
}))
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
switch (null)
|
||||
{
|
||||
default:
|
||||
url.searchParams
|
||||
.set(urlResolverFunction.paramName, descriptorsGroup
|
||||
.map((descriptor) => descriptor.id)
|
||||
.filter((descriptorId) => descriptorId)
|
||||
.join(urlResolverFunction.paramArraySeperator)
|
||||
)
|
||||
|
||||
const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
|
||||
if (!apiResponse.ok) break
|
||||
const values = followResponsePath<string[]>(await apiResponse.json(), urlResolverFunction.responsePath)
|
||||
|
||||
await Promise.all(values.map(async (value, index) =>
|
||||
{
|
||||
const descriptor = descriptorsGroup[index]
|
||||
if (value) results[descriptor.index] = value
|
||||
await LbryPathnameCache.put(value, descriptor.id)
|
||||
}))
|
||||
}
|
||||
progressCount += descriptorsGroup.length
|
||||
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
|
||||
}
|
||||
}
|
||||
|
||||
if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel'])
|
||||
if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video'])
|
||||
|
||||
}))
|
||||
|
||||
return results
|
||||
}
|
||||
141
src/common/yt/urlResolve.ts
Normal file
141
src/common/yt/urlResolve.ts
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
import { chunk, groupBy } from "lodash"
|
||||
import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YtUrlResolveResponsePath, ytUrlResolversSettings } from "../settings"
|
||||
import { LbryPathnameCache } from "./urlCache"
|
||||
|
||||
// const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS
|
||||
const QUERY_CHUNK_SIZE = 300
|
||||
|
||||
export interface YtIdResolverDescriptor
|
||||
{
|
||||
id: string
|
||||
type: 'channel' | 'video'
|
||||
}
|
||||
|
||||
/**
|
||||
* @param descriptorsWithIndex YT resource IDs to check
|
||||
* @returns a promise with the list of channels that were found on lbry
|
||||
*/
|
||||
export async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]>
|
||||
{
|
||||
const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({ ...descriptor, index }))
|
||||
descriptors = null as any
|
||||
const results: (string | null)[] = []
|
||||
|
||||
await Promise.all(descriptorsWithIndex.map(async (descriptor, index) =>
|
||||
{
|
||||
if (!descriptor) return
|
||||
const cache = await LbryPathnameCache.get(descriptor.id)
|
||||
|
||||
// Cache can be null, if there is no lbry url yet
|
||||
if (cache !== undefined)
|
||||
{
|
||||
// Directly setting it to results
|
||||
results[index] = cache
|
||||
|
||||
// We remove it so we dont ask it to API
|
||||
descriptorsWithIndex.splice(index, 1)
|
||||
}
|
||||
}))
|
||||
|
||||
const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE)
|
||||
let progressCount = 0
|
||||
await Promise.all(descriptorsChunks.map(async (descriptorChunk) =>
|
||||
{
|
||||
const descriptorsGroupedByType: Record<YtIdResolverDescriptor['type'], typeof descriptorsWithIndex | null> = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any
|
||||
|
||||
const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync()
|
||||
const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName]
|
||||
|
||||
const url = new URL(`https://${urlResolverSetting.hostname}`)
|
||||
|
||||
function followResponsePath<T>(response: any, responsePath: YtUrlResolveResponsePath)
|
||||
{
|
||||
for (const path of responsePath)
|
||||
{
|
||||
switch (typeof path)
|
||||
{
|
||||
case 'string':
|
||||
case 'number':
|
||||
response = response[path]
|
||||
break
|
||||
default:
|
||||
switch (path)
|
||||
{
|
||||
case Keys:
|
||||
response = Object.keys(response)
|
||||
break
|
||||
case Values:
|
||||
response = Object.values(response)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return response as T
|
||||
}
|
||||
|
||||
async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsWithIndex)
|
||||
{
|
||||
url.pathname = urlResolverFunction.pathname
|
||||
|
||||
if (urlResolverFunction.paramArraySeperator === SingleValueAtATime)
|
||||
{
|
||||
await Promise.all(descriptorsGroup.map(async (descriptor) =>
|
||||
{
|
||||
switch (null)
|
||||
{
|
||||
default:
|
||||
if (!descriptor.id) break
|
||||
url.searchParams.set(urlResolverFunction.paramName, descriptor.id)
|
||||
|
||||
const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
|
||||
if (!apiResponse.ok)
|
||||
{
|
||||
// Some API might not respond with 200 if it can't find the url
|
||||
if (apiResponse.status === 404) await LbryPathnameCache.put(null, descriptor.id)
|
||||
break
|
||||
}
|
||||
|
||||
const value = followResponsePath<string>(await apiResponse.json(), urlResolverFunction.responsePath)
|
||||
if (value) results[descriptor.index] = value
|
||||
await LbryPathnameCache.put(value, descriptor.id)
|
||||
}
|
||||
progressCount++
|
||||
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
|
||||
}))
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
switch (null)
|
||||
{
|
||||
default:
|
||||
url.searchParams
|
||||
.set(urlResolverFunction.paramName, descriptorsGroup
|
||||
.map((descriptor) => descriptor.id)
|
||||
.filter((descriptorId) => descriptorId)
|
||||
.join(urlResolverFunction.paramArraySeperator)
|
||||
)
|
||||
|
||||
const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
|
||||
if (!apiResponse.ok) break
|
||||
const values = followResponsePath<string[]>(await apiResponse.json(), urlResolverFunction.responsePath)
|
||||
|
||||
await Promise.all(values.map(async (value, index) =>
|
||||
{
|
||||
const descriptor = descriptorsGroup[index]
|
||||
if (value) results[descriptor.index] = value
|
||||
await LbryPathnameCache.put(value, descriptor.id)
|
||||
}))
|
||||
}
|
||||
progressCount += descriptorsGroup.length
|
||||
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
|
||||
}
|
||||
}
|
||||
|
||||
if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel'])
|
||||
if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video'])
|
||||
|
||||
}))
|
||||
|
||||
return results
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { parseProtocolUrl } from '../common/lbry-url'
|
||||
import { resolveById, YtIdResolverDescriptor } from '../common/yt'
|
||||
import { resolveById, YtIdResolverDescriptor } from '../common/yt/urlResolve'
|
||||
async function resolveYT(descriptor: YtIdResolverDescriptor) {
|
||||
const lbryProtocolUrl: string | null = await resolveById([descriptor]).then(a => a[0]);
|
||||
const segments = parseProtocolUrl(lbryProtocolUrl || '', { encode: true });
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import { h, render } from 'preact'
|
||||
import { useState } from 'preact/hooks'
|
||||
import { getExtensionSettingsAsync, targetPlatformSettings } from '../common/settings'
|
||||
import { getFileContent, getSubsFromCsv, getSubsFromJson, getSubsFromOpml, resolveById } from '../common/yt'
|
||||
import { getFileContent, getSubsFromCsv, getSubsFromJson, getSubsFromOpml } from '../common/yt'
|
||||
import { resolveById } from '../common/yt/urlResolve'
|
||||
import readme from './README.md'
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue