From da5cc12a7b9ad99e32ed7c3d947c6b0306a73cbc Mon Sep 17 00:00:00 2001 From: Kevin Raoofi Date: Tue, 6 Oct 2020 01:57:48 -0400 Subject: [PATCH] ytService and rewrite of YTtoLBRY YTtoLBRY: * Use DOMParser for reading OPML * OPML parsing was quite brittle as it relied on string parsing * Removed dead script inclusion in the HTML page * Removed global state variables * Use fetch to clean up the query logic * Promisified file, API calls, and settings querying to simplify logic Many of the utilities involving APIs were isolated to a common module. Particularly, ytService contains all methods for extracting IDs, querying for the lbry URLs, parsing OPML, and more. This functionally is heavily used in YTtoLBRY and lightly used by tabsOnUpdated. --- package-lock.json | 6 +++ package.json | 2 + src/common/yt.ts | 97 ++++++++++++++++++++++++++++++++++++ src/scripts/tabOnUpdated.ts | 37 ++------------ src/tools/YTtoLBRY.html | 2 +- src/tools/YTtoLBRY.js | 98 ------------------------------------- src/tools/YTtoLBRY.ts | 36 ++++++++++++++ 7 files changed, 146 insertions(+), 132 deletions(-) create mode 100644 src/common/yt.ts delete mode 100644 src/tools/YTtoLBRY.js create mode 100644 src/tools/YTtoLBRY.ts diff --git a/package-lock.json b/package-lock.json index 7c19896..9d54c00 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1736,6 +1736,12 @@ "integrity": "sha512-iUxzm1meBm3stxUMzRqgOVHjj4Kgpgu5w9fm4X7kPRfSgVRzythsucEN7/jtOo8SQzm+HfcxWWzJS0mJDH/3DQ==", "dev": true }, + "@types/lodash": { + "version": "4.14.162", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.162.tgz", + "integrity": "sha512-alvcho1kRUnnD1Gcl4J+hK0eencvzq9rmzvFPRmP5rPHx9VVsJj6bKLTATPVf9ktgv4ujzh7T+XWKp+jhuODig==", + "dev": true + }, "@types/minimatch": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", diff --git a/package.json b/package.json index 405114a..f0c995d 100644 --- a/package.json +++ b/package.json @@ -19,8 +19,10 @@ "dependencies": {}, "devDependencies": { "@types/chrome": "0.0.124", + "@types/lodash": "^4.14.162", "cpx": "^1.5.0", "cross-env": "^7.0.2", + "lodash": "^4.17.20", "npm-run-all": "^4.1.5", "parcel-bundler": "^1.12.4", "braces": ">=2.3.1", diff --git a/src/common/yt.ts b/src/common/yt.ts new file mode 100644 index 0000000..31800a9 --- /dev/null +++ b/src/common/yt.ts @@ -0,0 +1,97 @@ +import { chunk, groupBy, pickBy } from 'lodash'; + +const LBRY_API_HOST = 'https://api.lbry.com'; +const QUERY_CHUNK_SIZE = 325; + +interface YtResolverResponse { + success: boolean + error: object | null + data: { + videos?: Record + channels?: Record + } +} + +/** + * @param file to load + * @returns a promise with the file as a string + */ +export function getFileContent(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.addEventListener('load', event => resolve(event.target?.result as string || '')); + reader.addEventListener('error', () => { + reader.abort(); + reject(new DOMException(`Could not read ${file.name}`)); + }); + reader.readAsText(file); + }); +} + +export interface YTDescriptor { + id: string + type: 'channel' | 'video' +} + +export const ytService = { + + /** + * Reads the array of YT channels from an OPML file + * + * @param opmlContents an opml file as as tring + * @returns the channel URLs + */ + readOpml(opmlContents: string): string[] { + const opml = new DOMParser().parseFromString(opmlContents, 'application/xml'); + return Array.from(opml.querySelectorAll('outline > outline')) + .map(outline => outline.getAttribute('xmlUrl')) + .filter((url): url is string => !!url); // we don't want it if it's empty + }, + + /** + * Extracts the channelID from a YT URL. + * + * Handles these two types of YT URLs: + * * /feeds/videos.xml?channel_id=* + * * /channel/* + */ + getChannelId(channelURL: string) { + const match = channelURL.match(/channel\/([^\s?]*)/); + return match ? match[1] : new URL(channelURL).searchParams.get('channel_id'); + }, + + /** Extracts the video ID from a YT URL */ + getVideoId(url: string) { + const regex = /watch\/?\?.*v=([^\s&]*)/; + const match = url.match(regex); + return match ? match[1] : null; // match[1] is the videoId + }, + + getId(url: string): YTDescriptor | null { + const videoId = ytService.getVideoId(url); + if (videoId) return { id: videoId, type: 'video' }; + const channelId = ytService.getChannelId(url); + if (channelId) return { id: channelId, type: 'channel' }; + return null; + }, + + /** + * @param descriptors YT resource IDs to check + * @returns a promise with the list of channels that were found on lbry + */ + async resolveById(...descriptors: YTDescriptor[]): Promise { + const descChunks = chunk(descriptors, QUERY_CHUNK_SIZE); + const responses: (YtResolverResponse | null)[] = await Promise.all(descChunks.map(descChunk => { + const groups = groupBy(descChunk, d => d.type); + const params = new URLSearchParams(pickBy({ + video_ids: groups['video']?.map(s => s.id).join(','), + channel_ids: groups['channel']?.map(s => s.id).join(','), + })); + return fetch(`${LBRY_API_HOST}/yt/resolve?${params}`) + .then(rsp => rsp.ok ? rsp.json() : null); + })); + + return responses.filter((rsp): rsp is YtResolverResponse => !!rsp) + .flatMap(rsp => [...Object.values(rsp.data.videos || {}), ...Object.values(rsp.data.channels || {})]); // flatten the results into a 1D array + }, +}; diff --git a/src/scripts/tabOnUpdated.ts b/src/scripts/tabOnUpdated.ts index 4e02b68..3d39bdc 100644 --- a/src/scripts/tabOnUpdated.ts +++ b/src/scripts/tabOnUpdated.ts @@ -1,4 +1,5 @@ import { getSettingsAsync, redirectDomains } from "../common/settings"; +import { ytService } from "../common/yt"; chrome.tabs.onUpdated.addListener(async (tabId, changeInfo, { url: tabUrl }) => { const { enabled, redirect } = await getSettingsAsync('enabled', 'redirect'); @@ -37,19 +38,12 @@ chrome.tabs.onUpdated.addListener(async (tabId, changeInfo, { url: tabUrl }) => } return; } - const { id, type } = getId(tabUrl); - if (!id) return; - - const url = `https://api.lbry.com/yt/resolve?${type}_ids=${id}`; - const response = await fetch(url, { headers: { 'Content-Type': 'application/json' } }); - const json = await response.json(); - console.log(json); - const title = json.data[`${type}s`][id]; + const descriptor = ytService.getId(tabUrl); + if (!descriptor) return; + const title = (await ytService.resolveById(descriptor))[0] if (!title) return; console.log(title); - - console.log(redirect); let newUrl; if (redirect === "lbry.tv") { newUrl = `${urlPrefix}${title.replace(/^lbry:\/\//, "").replace(/#/g, ":")}?src=watch-on-lbry`; @@ -58,26 +52,3 @@ chrome.tabs.onUpdated.addListener(async (tabId, changeInfo, { url: tabUrl }) => } chrome.tabs.update(tabId, { url: newUrl }); }); - -function getId(url) { - const videoId = getVideoId(url); - if (videoId) return { id: videoId, type: "video" }; - const channelId = getChannelId(url); - if (channelId) return { id: channelId, type: "channel" }; - return {}; // Equivalent of returning null -} - -function getVideoId(url) { - const regex = /watch\/?\?.*v=([^\s&]*)/; - const match = url.match(regex); - return match ? match[1] : null; // match[1] is the videoId -} - -function getChannelId(url) { - const regex = /channel\/([^\s?]*)/; - const match = url.match(regex); - return match ? match[1] : null; // match[1] is the channelId -} - -function getNewUrl(title) { -} diff --git a/src/tools/YTtoLBRY.html b/src/tools/YTtoLBRY.html index 63cf668..0b2389b 100644 --- a/src/tools/YTtoLBRY.html +++ b/src/tools/YTtoLBRY.html @@ -4,7 +4,7 @@ Subscribtion Converter - + diff --git a/src/tools/YTtoLBRY.js b/src/tools/YTtoLBRY.js deleted file mode 100644 index ea4c8f3..0000000 --- a/src/tools/YTtoLBRY.js +++ /dev/null @@ -1,98 +0,0 @@ -import { redirectDomains } from '../common/settings' - -console.log("YouTube To LBRY finder!"); -var ytChannelsString = ""; -var lbryChannelsString = ""; -let lbryArray = []; -var toCheck = []; -let tempJson = {"0":0}; -var subconv; -var goButton; -var lbryChannelList; - -window.addEventListener('load', (event) => { - subconv = document.getElementById("subconv"); - goButton = document.getElementById("go-button"); - lbryChannelList = document.getElementById("lbry-channel-list"); - - goButton.addEventListener('click', () => onGoClicked()); -}); - -function getFile(file) { - const reader = new FileReader(); - reader.addEventListener('load', (event) => { - var content = event.target.result; - content = content.replace('',''); - content = content.replace('',''); - content = content.replace('/>',' '); - splitChannels = content.split("="); - toCheck = []; - for (var i = 0; i <= splitChannels.length-1; i++) { - tempChannel = splitChannels[i]; - if (tempChannel.indexOf("outline text")>=29) { - toCheck[toCheck.length] = tempChannel.slice(0, tempChannel.indexOf("outline text")-5); - } - } - lbryAPIrequest(); - }); - reader.readAsText(file); -} - -function onGoClicked() { - if (subconv.files.length > 0) { - getFile(subconv.files[0]); - } -} - -function lbryAPIrequest() { - // Clear current channel list - while (lbryChannelList.lastElementChild) { - lbryChannelList.removeChild(lbryChannelList.lastElementChild); - } - - chrome.storage.local.get('redirect', redirect => { - validateChannels(toCheck, redirect.redirect, []); - }); -} - -function validateChannels(channels, redirect, validatedChannels) { - const requestSize = 325; - var channelsString = ""; - for (let i = 0; i < channels.length && i < requestSize; i++) { - channelsString += `${channelsString.length > 0 ? ',' : ''}${channels[i]}` - } - request = new XMLHttpRequest(); - request.open("GET", `https://api.lbry.com/yt/resolve?channel_ids={${channelsString}}`); - request.send(); - request.onload = () => { - if (request.status == 200) { - var testChannels = JSON.parse(request.responseText).data.channels; - Object.keys(testChannels).map((testChannelKey) => { - let testChannel = testChannels[testChannelKey]; - if (testChannel != null) { - let link = redirectDomains[redirect].prefix + testChannel; - validatedChannels.push(link); - let li = document.createElement('li'); - let a = document.createElement('a'); - a.href = link; - a.innerText = link; - li.appendChild(a); - lbryChannelList.appendChild(li); - } - }); - } - if (requestSize < channels.length) { - channels.splice(0, requestSize); - validateChannels(channels, redirect, validatedChannels); - } else if (validatedChannels.length === 0) { - let li = document.createElement('li'); - li.innerText = "No channels found :("; - lbryChannelList.appendChild(li); - } - } -} - -chrome.storage.local.get('redirect', redirect => { - console.log(redirect); -}) diff --git a/src/tools/YTtoLBRY.ts b/src/tools/YTtoLBRY.ts new file mode 100644 index 0000000..ebb2d5b --- /dev/null +++ b/src/tools/YTtoLBRY.ts @@ -0,0 +1,36 @@ +import { getSettingsAsync, redirectDomains } from '../common/settings'; +import { getFileContent, YTDescriptor, ytService } from '../common/yt'; + +/** + * Parses OPML file and queries the API for lbry channels + * + * @param file to read + * @returns a promise with the list of channels that were found on lbry + */ +async function lbryChannelsFromOpml(file: File): Promise { + const lbryUrls = await ytService.resolveById(...ytService.readOpml(await getFileContent(file)) + .map(url => ytService.getId(url)) + .filter((id): id is YTDescriptor => !!id)); + + const { redirect } = await getSettingsAsync('redirect'); + const urlPrefix = redirectDomains[redirect].prefix; + return lbryUrls.map(channel => urlPrefix + channel); +} + +window.addEventListener('load', () => { + const subconv = document.getElementById('subconv') as HTMLInputElement; + const goButton = document.getElementById('go-button')!; + const lbryChannelList = document.getElementById('lbry-channel-list')!; + goButton.addEventListener('click', async () => { + const files = subconv.files; + if (!files || files.length <= 0) return; + + const resultsList = await lbryChannelsFromOpml(files[0]); + + if (resultsList.length === 0) { + lbryChannelList.innerHTML = `
  • No channels found :(
    • `; + return; + } + lbryChannelList.innerHTML = resultsList.map(link => `
  • ${link}
    • `).join('\n'); + }); +});