ytService and rewrite of YTtoLBRY

YTtoLBRY:

* Use DOMParser for reading OPML
  * OPML parsing was quite brittle as it relied on string parsing
* Removed dead script inclusion in the HTML page
* Removed global state variables
* Use fetch to clean up the query logic
* Promisified file, API calls, and settings querying to simplify logic

Many of the utilities involving APIs were isolated to a common module.
Particularly, ytService contains all methods for extracting IDs,
querying for the lbry URLs, parsing OPML, and more.

This functionally is heavily used in YTtoLBRY and lightly used by
tabsOnUpdated.

package-lock.json

@@ -1736,6 +1736,12 @@
       "integrity": "sha512-iUxzm1meBm3stxUMzRqgOVHjj4Kgpgu5w9fm4X7kPRfSgVRzythsucEN7/jtOo8SQzm+HfcxWWzJS0mJDH/3DQ==",
       "dev": true
     },
+    "@types/lodash": {
+      "version": "4.14.162",
+      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.162.tgz",
+      "integrity": "sha512-alvcho1kRUnnD1Gcl4J+hK0eencvzq9rmzvFPRmP5rPHx9VVsJj6bKLTATPVf9ktgv4ujzh7T+XWKp+jhuODig==",
+      "dev": true
+    },
     "@types/minimatch": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",

package.json

@@ -19,8 +19,10 @@
   "dependencies": {},
   "devDependencies": {
     "@types/chrome": "0.0.124",
+    "@types/lodash": "^4.14.162",
     "cpx": "^1.5.0",
     "cross-env": "^7.0.2",
+    "lodash": "^4.17.20",
     "npm-run-all": "^4.1.5",
     "parcel-bundler": "^1.12.4",
     "braces": ">=2.3.1",

src/common/yt.ts

@@ -0,0 +1,97 @@
+import { chunk, groupBy, pickBy } from 'lodash';
+
+const LBRY_API_HOST = 'https://api.lbry.com';
+const QUERY_CHUNK_SIZE = 325;
+
+interface YtResolverResponse {
+  success: boolean
+  error: object | null
+  data: {
+    videos?: Record<string, string>
+    channels?: Record<string, string>
+  }
+}
+
+/**
+ * @param file to load
+ * @returns a promise with the file as a string
+ */
+export function getFileContent(file: File): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.addEventListener('load', event => resolve(event.target?.result as string || ''));
+    reader.addEventListener('error', () => {
+      reader.abort();
+      reject(new DOMException(`Could not read ${file.name}`));
+    });
+    reader.readAsText(file);
+  });
+}
+
+export interface YTDescriptor {
+  id: string
+  type: 'channel' | 'video'
+}
+
+export const ytService = {
+
+  /**
+   * Reads the array of YT channels from an OPML file
+   *
+   * @param opmlContents an opml file as as tring
+   * @returns the channel URLs
+   */
+  readOpml(opmlContents: string): string[] {
+    const opml = new DOMParser().parseFromString(opmlContents, 'application/xml');
+    return Array.from(opml.querySelectorAll('outline > outline'))
+      .map(outline => outline.getAttribute('xmlUrl'))
+      .filter((url): url is string => !!url); // we don't want it if it's empty
+  },
+
+  /**
+   * Extracts the channelID from a YT URL.
+   *
+   * Handles these two types of YT URLs:
+   *  * /feeds/videos.xml?channel_id=*
+   *  * /channel/*
+   */
+  getChannelId(channelURL: string) {
+    const match = channelURL.match(/channel\/([^\s?]*)/);
+    return match ? match[1] : new URL(channelURL).searchParams.get('channel_id');
+  },
+
+  /** Extracts the video ID from a YT URL */
+  getVideoId(url: string) {
+    const regex = /watch\/?\?.*v=([^\s&]*)/;
+    const match = url.match(regex);
+    return match ? match[1] : null; // match[1] is the videoId
+  },
+
+  getId(url: string): YTDescriptor | null {
+    const videoId = ytService.getVideoId(url);
+    if (videoId) return { id: videoId, type: 'video' };
+    const channelId = ytService.getChannelId(url);
+    if (channelId) return { id: channelId, type: 'channel' };
+    return null;
+  },
+
+  /**
+  * @param descriptors YT resource IDs to check
+  * @returns a promise with the list of channels that were found on lbry
+  */
+  async resolveById(...descriptors: YTDescriptor[]): Promise<string[]> {
+    const descChunks = chunk(descriptors, QUERY_CHUNK_SIZE);
+    const responses: (YtResolverResponse | null)[] = await Promise.all(descChunks.map(descChunk => {
+      const groups = groupBy(descChunk, d => d.type);
+      const params = new URLSearchParams(pickBy({
+        video_ids: groups['video']?.map(s => s.id).join(','),
+        channel_ids: groups['channel']?.map(s => s.id).join(','),
+      }));
+      return fetch(`${LBRY_API_HOST}/yt/resolve?${params}`)
+        .then(rsp => rsp.ok ? rsp.json() : null);
+    }));
+
+    return responses.filter((rsp): rsp is YtResolverResponse => !!rsp)
+      .flatMap(rsp => [...Object.values(rsp.data.videos || {}), ...Object.values(rsp.data.channels || {})]);  // flatten the results into a 1D array
+  },
+};

src/scripts/tabOnUpdated.ts

@@ -1,4 +1,5 @@
 import { getSettingsAsync, redirectDomains } from "../common/settings";
+import { ytService } from "../common/yt";
 
 chrome.tabs.onUpdated.addListener(async (tabId, changeInfo, { url: tabUrl }) => {
   const { enabled, redirect } = await getSettingsAsync('enabled', 'redirect');
@@ -37,19 +38,12 @@ chrome.tabs.onUpdated.addListener(async (tabId, changeInfo, { url: tabUrl }) =>
     }
     return;
   }
-  const { id, type } = getId(tabUrl);
-  if (!id) return;
-
-  const url = `https://api.lbry.com/yt/resolve?${type}_ids=${id}`;
-  const response = await fetch(url, { headers: { 'Content-Type': 'application/json' } });
-  const json = await response.json();
-  console.log(json);
-  const title = json.data[`${type}s`][id];
+  const descriptor = ytService.getId(tabUrl);
+  if (!descriptor) return;
+  const title = (await ytService.resolveById(descriptor))[0]
   if (!title) return;
   console.log(title);
 
-
-  console.log(redirect);
   let newUrl;
   if (redirect === "lbry.tv") {
     newUrl = `${urlPrefix}${title.replace(/^lbry:\/\//, "").replace(/#/g, ":")}?src=watch-on-lbry`;
@@ -58,26 +52,3 @@ chrome.tabs.onUpdated.addListener(async (tabId, changeInfo, { url: tabUrl }) =>
   }
   chrome.tabs.update(tabId, { url: newUrl });
 });
-
-function getId(url) {
-  const videoId = getVideoId(url);
-  if (videoId) return { id: videoId, type: "video" };
-  const channelId = getChannelId(url);
-  if (channelId) return { id: channelId, type: "channel" };
-  return {}; // Equivalent of returning null
-}
-
-function getVideoId(url) {
-  const regex = /watch\/?\?.*v=([^\s&]*)/;
-  const match = url.match(regex);
-  return match ? match[1] : null; // match[1] is the videoId
-}
-
-function getChannelId(url) {
-  const regex = /channel\/([^\s?]*)/;
-  const match = url.match(regex);
-  return match ? match[1] : null; // match[1] is the channelId
-}
-
-function getNewUrl(title) {
-}

src/tools/YTtoLBRY.html

@@ -4,7 +4,7 @@
     <link rel="stylesheet" href="YTtoLBRY.css" />
     <meta charset="utf-8">
     <title>Subscribtion Converter</title>
-    <script src="YTtoLBRY.js" charset="utf-8"></script>
+    <script src="YTtoLBRY.ts" charset="utf-8"></script>
   </head>
 
   <body style="background-color:#474747;">

src/tools/YTtoLBRY.js

@@ -1,98 +0,0 @@
-import { redirectDomains } from '../common/settings'
-
-console.log("YouTube To LBRY finder!");
-var ytChannelsString = "";
-var lbryChannelsString = "";
-let lbryArray = [];
-var toCheck = [];
-let tempJson = {"0":0};
-var subconv;
-var goButton;
-var lbryChannelList;
-
-window.addEventListener('load', (event) => {
-    subconv = document.getElementById("subconv");
-    goButton = document.getElementById("go-button");
-    lbryChannelList = document.getElementById("lbry-channel-list");
-
-    goButton.addEventListener('click', () => onGoClicked());
-});
-
-function getFile(file) {
-    const reader = new FileReader();
-    reader.addEventListener('load', (event) => {
-        var content = event.target.result;
-        content = content.replace('<opml version="1.1">','');
-        content = content.replace('<body>','');
-        content = content.replace('/><outline','');
-        content = content.replace('</outline></body></opml>',' ');
-        splitChannels = content.split("=");
-        toCheck = [];
-        for (var i = 0; i <= splitChannels.length-1; i++) {
-            tempChannel = splitChannels[i];
-            if (tempChannel.indexOf("outline text")>=29) {
-                toCheck[toCheck.length] = tempChannel.slice(0, tempChannel.indexOf("outline text")-5);
-            }
-        }
-        lbryAPIrequest();
-    });
-    reader.readAsText(file);
-}
-
-function onGoClicked() {
-    if (subconv.files.length > 0) {
-        getFile(subconv.files[0]);
-    }
-}
-
-function lbryAPIrequest() {
-    // Clear current channel list
-    while (lbryChannelList.lastElementChild) {
-        lbryChannelList.removeChild(lbryChannelList.lastElementChild);
-    }
-
-    chrome.storage.local.get('redirect', redirect => {
-        validateChannels(toCheck, redirect.redirect, []);
-    });
-}
-
-function validateChannels(channels, redirect, validatedChannels) {
-    const requestSize = 325;
-    var channelsString = "";
-    for (let i = 0; i < channels.length && i < requestSize; i++) {
-        channelsString += `${channelsString.length > 0 ? ',' : ''}${channels[i]}`
-    }
-    request = new XMLHttpRequest();
-    request.open("GET", `https://api.lbry.com/yt/resolve?channel_ids={${channelsString}}`);
-    request.send();
-    request.onload = () => {
-        if (request.status == 200) {
-            var testChannels = JSON.parse(request.responseText).data.channels;
-            Object.keys(testChannels).map((testChannelKey) => {
-                let testChannel = testChannels[testChannelKey];
-                if (testChannel != null) {
-                    let link = redirectDomains[redirect].prefix + testChannel;
-                    validatedChannels.push(link);
-                    let li = document.createElement('li');
-                    let a = document.createElement('a');
-                    a.href = link;
-                    a.innerText = link;
-                    li.appendChild(a);
-                    lbryChannelList.appendChild(li);
-                }
-            });
-        }
-        if (requestSize < channels.length) {
-            channels.splice(0, requestSize);
-            validateChannels(channels, redirect, validatedChannels);
-        } else if (validatedChannels.length === 0) {
-            let li = document.createElement('li');
-            li.innerText = "No channels found :(";
-            lbryChannelList.appendChild(li);
-        }
-    }
-}
-
-chrome.storage.local.get('redirect', redirect => {
-    console.log(redirect);
-})

src/tools/YTtoLBRY.ts

@@ -0,0 +1,36 @@
+import { getSettingsAsync, redirectDomains } from '../common/settings';
+import { getFileContent, YTDescriptor, ytService } from '../common/yt';
+
+/**
+ * Parses OPML file and queries the API for lbry channels
+ *
+ * @param file to read
+ * @returns a promise with the list of channels that were found on lbry
+ */
+async function lbryChannelsFromOpml(file: File): Promise<string[]> {
+  const lbryUrls = await ytService.resolveById(...ytService.readOpml(await getFileContent(file))
+    .map(url => ytService.getId(url))
+    .filter((id): id is YTDescriptor => !!id));
+
+  const { redirect } = await getSettingsAsync('redirect');
+  const urlPrefix = redirectDomains[redirect].prefix;
+  return lbryUrls.map(channel => urlPrefix + channel);
+}
+
+window.addEventListener('load', () => {
+  const subconv = document.getElementById('subconv') as HTMLInputElement;
+  const goButton = document.getElementById('go-button')!;
+  const lbryChannelList = document.getElementById('lbry-channel-list')!;
+  goButton.addEventListener('click', async () => {
+    const files = subconv.files;
+    if (!files || files.length <= 0) return;
+
+    const resultsList = await lbryChannelsFromOpml(files[0]);
+
+    if (resultsList.length === 0) {
+      lbryChannelList.innerHTML = `<li>No channels found :(</li>`;
+      return;
+    }
+    lbryChannelList.innerHTML = resultsList.map(link => `<li><a href='${link}'>${link}</a></li>`).join('\n');
+  });
+});