🥞 Refactor

This commit is contained in:
Shiba 2022-01-09 19:10:27 +00:00
parent 75cb9cf01d
commit cb4b4f4b2e
4 changed files with 239 additions and 230 deletions

View file

@ -1,23 +1,26 @@
import chunk from 'lodash/chunk' import chunk from 'lodash/chunk'
import groupBy from 'lodash/groupBy' import groupBy from 'lodash/groupBy'
import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YtUrlResolveResponsePath, ytUrlResolversSettings } from '../settings' import { getExtensionSettingsAsync, Keys, SingleValueAtATime, Values, YtUrlResolveFunction, YtUrlResolveResponsePath, ytUrlResolversSettings } from '../settings'
import { LbryURLCache } from './urlCache'
// const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS // const LBRY_API_HOST = 'https://api.odysee.com'; MOVED TO SETTINGS
const QUERY_CHUNK_SIZE = 300; const QUERY_CHUNK_SIZE = 300
interface YtExportedJsonSubscription { interface YtExportedJsonSubscription
id: string; {
etag: string; id: string
title: string; etag: string
title: string
snippet: { snippet: {
description: string; description: string
resourceId: { resourceId: {
channelId: string; channelId: string
}; }
}; }
} }
export interface YtIdResolverDescriptor { export interface YtIdResolverDescriptor
{
id: string id: string
type: 'channel' | 'video' type: 'channel' | 'video'
} }
@ -26,19 +29,21 @@ export interface YtIdResolverDescriptor {
* @param file to load * @param file to load
* @returns a promise with the file as a string * @returns a promise with the file as a string
*/ */
export function getFileContent(file: File): Promise<string> { export function getFileContent(file: File): Promise<string>
return new Promise((resolve, reject) => { {
const reader = new FileReader(); return new Promise((resolve, reject) =>
reader.addEventListener('load', event => resolve(event.target?.result as string || '')); {
reader.addEventListener('error', () => { const reader = new FileReader()
reader.abort(); reader.addEventListener('load', event => resolve(event.target?.result as string || ''))
reject(new DOMException(`Could not read ${file.name}`)); reader.addEventListener('error', () =>
}); {
reader.readAsText(file); reader.abort()
}); reject(new DOMException(`Could not read ${file.name}`))
})
reader.readAsText(file)
})
} }
export const ytService = (() => {
/** /**
* Extracts the channelID from a YT URL. * Extracts the channelID from a YT URL.
* *
@ -46,9 +51,10 @@ export const ytService = (() => {
* * /feeds/videos.xml?channel_id=* * * /feeds/videos.xml?channel_id=*
* * /channel/* * * /channel/*
*/ */
function getChannelId(channelURL: string) { export function getChannelId(channelURL: string)
const match = channelURL.match(/channel\/([^\s?]*)/); {
return match ? match[1] : new URL(channelURL).searchParams.get('channel_id'); const match = channelURL.match(/channel\/([^\s?]*)/)
return match ? match[1] : new URL(channelURL).searchParams.get('channel_id')
} }
/** /**
@ -57,14 +63,15 @@ export const ytService = (() => {
* @param opmlContents an opml file as as tring * @param opmlContents an opml file as as tring
* @returns the channel IDs * @returns the channel IDs
*/ */
function readOpml(opmlContents: string): string[] { export function getSubsFromOpml(opmlContents: string): string[]
const opml = new DOMParser().parseFromString(opmlContents, 'application/xml'); {
const opml = new DOMParser().parseFromString(opmlContents, 'application/xml')
opmlContents = '' opmlContents = ''
return Array.from(opml.querySelectorAll('outline > outline')) return Array.from(opml.querySelectorAll('outline > outline'))
.map(outline => outline.getAttribute('xmlUrl')) .map(outline => outline.getAttribute('xmlUrl'))
.filter((url): url is string => !!url) .filter((url): url is string => !!url)
.map(url => getChannelId(url)) .map(url => getChannelId(url))
.filter((url): url is string => !!url); // we don't want it if it's empty .filter((url): url is string => !!url) // we don't want it if it's empty
} }
/** /**
@ -73,10 +80,11 @@ export const ytService = (() => {
* @param jsonContents a JSON file as a string * @param jsonContents a JSON file as a string
* @returns the channel IDs * @returns the channel IDs
*/ */
function readJson(jsonContents: string): string[] { export function getSubsFromJson(jsonContents: string): string[]
const subscriptions: YtExportedJsonSubscription[] = JSON.parse(jsonContents); {
const subscriptions: YtExportedJsonSubscription[] = JSON.parse(jsonContents)
jsonContents = '' jsonContents = ''
return subscriptions.map(sub => sub.snippet.resourceId.channelId); return subscriptions.map(sub => sub.snippet.resourceId.channelId)
} }
/** /**
@ -85,80 +93,31 @@ export const ytService = (() => {
* @param csvContent a CSV file as a string * @param csvContent a CSV file as a string
* @returns the channel IDs * @returns the channel IDs
*/ */
function readCsv(csvContent: string): string[] { export function getSubsFromCsv(csvContent: string): string[]
{
const rows = csvContent.split('\n') const rows = csvContent.split('\n')
csvContent = '' csvContent = ''
return rows.slice(1).map((row) => row.substring(0, row.indexOf(','))) return rows.slice(1).map((row) => row.substring(0, row.indexOf(',')))
} }
const URLResolverCache = (() =>
{
const openRequest = self.indexedDB?.open("yt-url-resolver-cache")
if (openRequest)
{
openRequest.addEventListener('upgradeneeded', () => openRequest.result.createObjectStore("store").createIndex("expireAt", "expireAt"))
// Delete Expired
openRequest.addEventListener('success', () =>
{
const transaction = openRequest.result.transaction("store", "readwrite")
const range = IDBKeyRange.upperBound(new Date())
const expireAtCursorRequest = transaction.objectStore("store").index("expireAt").openCursor(range)
expireAtCursorRequest.addEventListener('success', () =>
{
const expireCursor = expireAtCursorRequest.result
if (!expireCursor) return
expireCursor.delete()
expireCursor.continue()
})
})
}
else console.warn(`IndexedDB not supported`)
async function put(url: string | null, id: string): Promise<void>
{
return await new Promise((resolve, reject) =>
{
const store = openRequest.result.transaction("store", "readwrite").objectStore("store")
if (!store) return resolve()
const request = store.put({ value: url, expireAt: new Date(Date.now() + 24 * 60 * 60 * 1000) }, id)
request.addEventListener('success', () => resolve())
request.addEventListener('error', () => reject(request.error))
})
}
async function get(id: string): Promise<string | null>
{
return (await new Promise((resolve, reject) =>
{
const store = openRequest.result.transaction("store", "readonly").objectStore("store")
if (!store) return resolve(null)
const request = store.get(id)
request.addEventListener('success', () => resolve(request.result))
request.addEventListener('error', () => reject(request.error))
}) as any)?.value
}
return { put, get }
})()
/** /**
* @param descriptorsWithIndex YT resource IDs to check * @param descriptorsWithIndex YT resource IDs to check
* @returns a promise with the list of channels that were found on lbry * @returns a promise with the list of channels that were found on lbry
*/ */
async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]> { export async function resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]>
{
const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({ ...descriptor, index })) const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({ ...descriptor, index }))
descriptors = null as any descriptors = null as any
const results: (string | null)[] = [] const results: (string | null)[] = []
await Promise.all(descriptorsWithIndex.map(async (descriptor, index) => { await Promise.all(descriptorsWithIndex.map(async (descriptor, index) =>
{
if (!descriptor) return if (!descriptor) return
const cache = await URLResolverCache.get(descriptor.id) const cache = await LbryURLCache.get(descriptor.id)
// Cache can be null, if there is no lbry url yet // Cache can be null, if there is no lbry url yet
if (cache !== undefined) { if (cache !== undefined)
{
// Directly setting it to results // Directly setting it to results
results[index] = cache results[index] = cache
@ -167,16 +126,16 @@ export const ytService = (() => {
} }
})) }))
const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE); const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE)
let progressCount = 0; let progressCount = 0
await Promise.all(descriptorsChunks.map(async (descriptorChunk) => await Promise.all(descriptorsChunks.map(async (descriptorChunk) =>
{ {
const descriptorsGroupedByType: Record<YtIdResolverDescriptor['type'], typeof descriptorsWithIndex | null> = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any; const descriptorsGroupedByType: Record<YtIdResolverDescriptor['type'], typeof descriptorsWithIndex | null> = groupBy(descriptorChunk, (descriptor) => descriptor.type) as any
const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync() const { urlResolver: urlResolverSettingName } = await getExtensionSettingsAsync()
const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName] const urlResolverSetting = ytUrlResolversSettings[urlResolverSettingName]
const url = new URL(`https://${urlResolverSetting.hostname}`); const url = new URL(`https://${urlResolverSetting.hostname}`)
function followResponsePath<T>(response: any, responsePath: YtUrlResolveResponsePath) function followResponsePath<T>(response: any, responsePath: YtUrlResolveResponsePath)
{ {
@ -209,23 +168,25 @@ export const ytService = (() => {
if (urlResolverFunction.paramArraySeperator === SingleValueAtATime) if (urlResolverFunction.paramArraySeperator === SingleValueAtATime)
{ {
await Promise.all(descriptorsGroup.map(async (descriptor) => { await Promise.all(descriptorsGroup.map(async (descriptor) =>
{
switch (null) switch (null)
{ {
default: default:
if (!descriptor.id) break if (!descriptor.id) break
url.searchParams.set(urlResolverFunction.paramName, descriptor.id) url.searchParams.set(urlResolverFunction.paramName, descriptor.id)
const apiResponse = await fetch(url.toString(), { cache: 'no-store' }); const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
if (!apiResponse.ok) { if (!apiResponse.ok)
{
// Some API might not respond with 200 if it can't find the url // Some API might not respond with 200 if it can't find the url
if (apiResponse.status === 404) await URLResolverCache.put(null, descriptor.id) if (apiResponse.status === 404) await LbryURLCache.put(null, descriptor.id)
break break
} }
const value = followResponsePath<string>(await apiResponse.json(), urlResolverFunction.responsePath) const value = followResponsePath<string>(await apiResponse.json(), urlResolverFunction.responsePath)
if (value) results[descriptor.index] = value if (value) results[descriptor.index] = value
await URLResolverCache.put(value, descriptor.id) await LbryURLCache.put(value, descriptor.id)
} }
progressCount++ progressCount++
if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length) if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length)
@ -244,14 +205,15 @@ export const ytService = (() => {
.join(urlResolverFunction.paramArraySeperator) .join(urlResolverFunction.paramArraySeperator)
) )
const apiResponse = await fetch(url.toString(), { cache: 'no-store' }); const apiResponse = await fetch(url.toString(), { cache: 'no-store' })
if (!apiResponse.ok) break if (!apiResponse.ok) break
const values = followResponsePath<string[]>(await apiResponse.json(), urlResolverFunction.responsePath) const values = followResponsePath<string[]>(await apiResponse.json(), urlResolverFunction.responsePath)
await Promise.all(values.map(async (value, index) => { await Promise.all(values.map(async (value, index) =>
{
const descriptor = descriptorsGroup[index] const descriptor = descriptorsGroup[index]
if (value) results[descriptor.index] = value if (value) results[descriptor.index] = value
await URLResolverCache.put(value, descriptor.id) await LbryURLCache.put(value, descriptor.id)
})) }))
} }
progressCount += descriptorsGroup.length progressCount += descriptorsGroup.length
@ -262,10 +224,7 @@ export const ytService = (() => {
if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel']) if (descriptorsGroupedByType['channel']) await requestGroup(urlResolverSetting.functions.getChannelId, descriptorsGroupedByType['channel'])
if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video']) if (descriptorsGroupedByType['video']) await requestGroup(urlResolverSetting.functions.getVideoId, descriptorsGroupedByType['video'])
})); }))
return results return results
} }
return { readCsv, readJson, readOpml, resolveById }
})()

50
src/common/yt/urlCache.ts Normal file
View file

@ -0,0 +1,50 @@
const openRequest = self.indexedDB?.open("yt-url-resolver-cache")
if (openRequest)
{
openRequest.addEventListener('upgradeneeded', () => openRequest.result.createObjectStore("store").createIndex("expireAt", "expireAt"))
// Delete Expired
openRequest.addEventListener('success', () =>
{
const transaction = openRequest.result.transaction("store", "readwrite")
const range = IDBKeyRange.upperBound(new Date())
const expireAtCursorRequest = transaction.objectStore("store").index("expireAt").openCursor(range)
expireAtCursorRequest.addEventListener('success', () =>
{
const expireCursor = expireAtCursorRequest.result
if (!expireCursor) return
expireCursor.delete()
expireCursor.continue()
})
})
}
else console.warn(`IndexedDB not supported`)
async function put(url: string | null, id: string): Promise<void>
{
return await new Promise((resolve, reject) =>
{
const store = openRequest.result.transaction("store", "readwrite").objectStore("store")
if (!store) return resolve()
const request = store.put({ value: url, expireAt: new Date(Date.now() + 24 * 60 * 60 * 1000) }, id)
request.addEventListener('success', () => resolve())
request.addEventListener('error', () => reject(request.error))
})
}
async function get(id: string): Promise<string | null>
{
return (await new Promise((resolve, reject) =>
{
const store = openRequest.result.transaction("store", "readonly").objectStore("store")
if (!store) return resolve(null)
const request = store.get(id)
request.addEventListener('success', () => resolve(request.result))
request.addEventListener('error', () => reject(request.error))
}) as any)?.value
}
export const LbryURLCache = { put, get }

View file

@ -1,7 +1,7 @@
import { ExtensionSettings, getExtensionSettingsAsync, getSourcePlatfromSettingsFromHostname, TargetPlatformName, targetPlatformSettings } from '../common/settings'
import type { UpdateContext } from '../scripts/tabOnUpdated'
import { h, JSX, render } from 'preact' import { h, JSX, render } from 'preact'
import { YtIdResolverDescriptor, ytService } from '../common/yt' import { getExtensionSettingsAsync, getSourcePlatfromSettingsFromHostname, TargetPlatformName, targetPlatformSettings } from '../common/settings'
import { resolveById, YtIdResolverDescriptor } from '../common/yt'
import type { UpdateContext } from '../scripts/tabOnUpdated'
const sleep = (t: number) => new Promise(resolve => setTimeout(resolve, t)); const sleep = (t: number) => new Promise(resolve => setTimeout(resolve, t));
@ -177,7 +177,7 @@ window.addEventListener('load', async () =>
const videoId = url.searchParams.get('v') const videoId = url.searchParams.get('v')
if (!videoId) return if (!videoId) return
const descriptor: YtIdResolverDescriptor = { id: videoId, type: 'video' } const descriptor: YtIdResolverDescriptor = { id: videoId, type: 'video' }
const lbryPathname = (await ytService.resolveById([descriptor]))[0] const lbryPathname = (await resolveById([descriptor]))[0]
if (!lbryPathname) return if (!lbryPathname) return
updateButton({ descriptor, lbryPathname, redirect: settings.redirect, targetPlatform: settings.targetPlatform }) updateButton({ descriptor, lbryPathname, redirect: settings.redirect, targetPlatform: settings.targetPlatform })
} }

View file

@ -1,7 +1,7 @@
import { h, render } from 'preact' import { h, render } from 'preact'
import { useState } from 'preact/hooks' import { useState } from 'preact/hooks'
import { getExtensionSettingsAsync, targetPlatformSettings } from '../common/settings' import { getExtensionSettingsAsync, targetPlatformSettings } from '../common/settings'
import { getFileContent, ytService } from '../common/yt' import { getFileContent, getSubsFromCsv, getSubsFromJson, getSubsFromOpml, resolveById } from '../common/yt'
import readme from './README.md' import readme from './README.md'
@ -16,10 +16,10 @@ async function lbryChannelsFromFile(file: File) {
const ext = file.name.split('.').pop()?.toLowerCase(); const ext = file.name.split('.').pop()?.toLowerCase();
const ids = new Set(( const ids = new Set((
ext === 'xml' || ext == 'opml' ? ytService.readOpml : ext === 'xml' || ext == 'opml' ? getSubsFromOpml :
ext === 'csv' ? ytService.readCsv : ext === 'csv' ? getSubsFromCsv :
ytService.readJson)(await getFileContent(file))) getSubsFromJson)(await getFileContent(file)))
const lbryUrls = await ytService.resolveById( const lbryUrls = await resolveById(
Array.from(ids).map(id => ({ id, type: 'channel' } as const)), Array.from(ids).map(id => ({ id, type: 'channel' } as const)),
(progress) => render(<YTtoLBRY progress={progress} />, document.getElementById('root')!)); (progress) => render(<YTtoLBRY progress={progress} />, document.getElementById('root')!));
const { targetPlatform: platform } = await getExtensionSettingsAsync(); const { targetPlatform: platform } = await getExtensionSettingsAsync();