diff --git a/package.json b/package.json index 72455ac19..d387e3456 100644 --- a/package.json +++ b/package.json @@ -54,12 +54,12 @@ "electron-notarize": "^1.0.0", "electron-updater": "^4.2.4", "express": "^4.17.1", - "feed": "^4.2.2", "if-env": "^1.0.4", "react-datetime-picker": "^3.2.1", "react-plastic": "^1.1.1", "react-top-loading-bar": "^2.0.1", "remove-markdown": "^0.3.0", + "rss": "^1.2.2", "source-map-explorer": "^2.5.2", "tempy": "^0.6.0", "videojs-contrib-ads": "^6.9.0", diff --git a/web/src/routes.js b/web/src/routes.js index d41d1a753..d3e91f631 100644 --- a/web/src/routes.js +++ b/web/src/routes.js @@ -19,9 +19,11 @@ function getStreamUrl(ctx) { } const rssMiddleware = async (ctx) => { - const xml = await getRss(ctx); - ctx.set('Content-Type', 'application/rss+xml'); - ctx.body = xml; + const rss = await getRss(ctx); + if (rss.startsWith(' { diff --git a/web/src/rss.js b/web/src/rss.js index 98684ab89..9bd1a07b7 100644 --- a/web/src/rss.js +++ b/web/src/rss.js @@ -1,7 +1,7 @@ const { generateDownloadUrl } = require('../../ui/util/web'); -const { URL, SITE_NAME, LBRY_WEB_API, FAVICON } = require('../../config.js'); +const { URL, SITE_NAME, LBRY_WEB_API } = require('../../config.js'); const { Lbry } = require('lbry-redux'); -const Feed = require('feed').Feed; +const Rss = require('rss'); const SDK_API_PATH = `${LBRY_WEB_API}/api/v1`; const proxyURL = `${SDK_API_PATH}/proxy`; @@ -9,6 +9,10 @@ Lbry.setDaemonConnectionString(proxyURL); const NUM_ENTRIES = 500; +// **************************************************************************** +// Fetch claim info +// **************************************************************************** + async function doClaimSearch(options) { let results; try { @@ -19,15 +23,21 @@ async function doClaimSearch(options) { async function getChannelClaim(name, claimId) { let claim; + let error; + try { const url = `lbry://${name}#${claimId}`; const response = await Lbry.resolve({ urls: [url] }); - if (response && response[url] && !response[url].error) { claim = response && response[url]; } } catch {} - return claim || 'The RSS URL is invalid or is not associated with any channel.'; + + if (!claim) { + error = 'The RSS URL is invalid or is not associated with any channel.'; + } + + return { claim, error }; } async function getClaimsFromChannel(claimId, count) { @@ -43,103 +53,218 @@ async function getClaimsFromChannel(claimId, count) { return await doClaimSearch(options); } -async function getFeed(channelClaim, feedLink) { - const replaceLineFeeds = (str) => str.replace(/(?:\r\n|\r|\n)/g, '
'); +// **************************************************************************** +// Helpers +// **************************************************************************** - const fmtDescription = (description) => replaceLineFeeds(description); +const generateEnclosureForClaimContent = (claim) => { + const value = claim.value; + if (!value || !value.stream_type) { + return undefined; + } - const sanitizeThumbsUrl = (url) => { - if (typeof url === 'string' && url.startsWith('https://')) { - return encodeURI(url).replace(/&/g, '%26'); - } - return ''; - }; + switch (value.stream_type) { + case 'video': + case 'audio': + case 'image': + case 'document': + case 'software': + return { + url: generateDownloadUrl(claim.name, claim.claim_id), + type: (value.source && value.source.media_type) || undefined, + size: (value.source && value.source.size) || 0, // Per spec, 0 is a valid fallback. + }; - const getEnclosure = (claim) => { - const value = claim.value; - if (!value || !value.stream_type || !value.source || !value.source.media_type) { + default: return undefined; + } +}; + +const getLanguageValue = (claim) => { + if (claim && claim.value && claim.value.languages && claim.value.languages.length > 0) { + return claim.value.languages[0]; + } + return 'en'; +}; + +const replaceLineFeeds = (str) => str.replace(/(?:\r\n|\r|\n)/g, '
'); + +const isEmailRoughlyValid = (email) => /^\S+@\S+$/.test(email); + +/** + * 'itunes:owner' is required by castfeedvalidator (w3c allows omission), and + * both name and email must be defined. The email must also be a "valid" one. + * + * Use a fallback email when the creator did not specify one. The email will not + * be shown to the user; it is just used for administrative purposes. + * + * @param claim + * @returns any + */ +const generateItunesOwnerElement = (claim) => { + let name = '---'; + let email = 'no-reply@odysee.com'; + + if (claim && claim.value) { + name = claim.name; + if (isEmailRoughlyValid(claim.value.email)) { + email = claim.value.email; } + } - switch (value.stream_type) { - case 'video': - case 'audio': - case 'image': - case 'document': - case 'software': - return { - url: encodeURI(generateDownloadUrl(claim.name, claim.claim_id)), - type: value.source.media_type, - length: value.source.size || 0, // Per spec, 0 is a valid fallback. - }; + return { + 'itunes:owner': [{ 'itunes:name': name }, { 'itunes:email': email }], + }; +}; - default: - return undefined; +const generateItunesExplicitElement = (claim) => { + const tags = (claim && claim.value && claim.tags) || []; + return { 'itunes:explicit': tags.includes('mature') ? 'yes' : 'no' }; +}; + +const getItunesCategory = (claim) => { + const itunesCategories = [ + 'Arts', + 'Business', + 'Comedy', + 'Education', + 'Fiction', + 'Government', + 'History', + 'Health & Fitness', + 'Kids & Family', + 'Leisure', + 'Music', + 'News', + 'Religion & Spirituality', + 'Science', + 'Society & Culture', + 'Sports', + 'Technology', + 'True Crime', + 'TV & Film', + ]; + + const tags = (claim && claim.value && claim.tags) || []; + for (let i = 0; i < tags.length; ++i) { + const tag = tags[i]; + if (itunesCategories.includes(tag)) { + // "Note: Although you can specify more than one category and subcategory + // in your RSS feed, Apple Podcasts only recognizes the first category and + // subcategory." + // --> The only parse the first found tag. + return tag.replace('&', '&'); } - }; + } - const value = channelClaim.value; - const title = value ? value.title : channelClaim.name; + // itunes will not accept any other categories, and the element is required + // to pass castfeedvalidator. So, fallback to 'Leisure' (closes to "General") + // if the creator did not specify a tag. + return 'Leisure'; +}; - const options = { - favicon: FAVICON || URL + '/public/favicon.png', - generator: SITE_NAME + ' RSS Feed', - title: title + ' on ' + SITE_NAME, - description: fmtDescription(value && value.description ? value.description : ''), - link: encodeURI(`${URL}/${channelClaim.name}:${channelClaim.claim_id}`), - image: sanitizeThumbsUrl(value && value.thumbnail ? value.thumbnail.url : ''), - feedLinks: { - rss: encodeURI(feedLink), - }, - author: { - name: encodeURI(channelClaim.name), - link: encodeURI(URL + '/' + channelClaim.name + ':' + channelClaim.claim_id), - }, - }; +const generateItunesDurationElement = (claim) => { + let duration; + if (claim && claim.value) { + if (claim.value.video) { + duration = claim.value.video.duration; + } else if (claim.value.audio) { + duration = claim.value.audio.duration; + } + } - const feed = new Feed(options); - const latestClaims = await getClaimsFromChannel(channelClaim.claim_id, NUM_ENTRIES); + if (duration) { + return { 'itunes:duration': `${duration}` }; + } +}; - latestClaims.forEach((c) => { - const meta = c.meta; - const value = c.value; +const generateItunesImageElement = (claim) => { + const thumbnailUrl = (claim && claim.value && claim.value.thumbnail && claim.value.thumbnail.url) || ''; + if (thumbnailUrl) { + return { + 'itunes:image': { _attr: { href: thumbnailUrl } }, + }; + } +}; - const title = value && value.title ? value.title : c.name; - const thumbnailUrl = value && value.thumbnail ? value.thumbnail.url : ''; - const thumbnailHtml = thumbnailUrl ? `

thumbnail

` : ''; +const getFormattedDescription = (claim) => { + return replaceLineFeeds((claim && claim.value && claim.value.description) || ''); +}; - feed.addItem({ - id: c.claim_id, - guid: encodeURI(URL + '/' + c.name + ':' + c.claim_id), - title: value && value.title ? value.title : c.name, - description: thumbnailHtml + fmtDescription(value && value.description ? value.description : ''), - link: encodeURI(URL + '/' + c.name + ':' + c.claim_id), - date: new Date(meta ? meta.creation_timestamp * 1000 : null), - enclosure: getEnclosure(c), +// **************************************************************************** +// Generate +// **************************************************************************** + +function generateFeed(feedLink, channelClaim, claimsInChannel) { + // --- Channel --- + const feed = new Rss({ + title: ((channelClaim.value && channelClaim.value.title) || channelClaim.name) + ' on ' + SITE_NAME, + description: getFormattedDescription(channelClaim), + feed_url: feedLink, + site_url: URL, + image_url: (channelClaim.value && channelClaim.value.thumbnail && channelClaim.value.thumbnail.url) || undefined, + language: getLanguageValue(channelClaim), + custom_namespaces: { itunes: 'http://www.itunes.com/dtds/podcast-1.0.dtd' }, + custom_elements: [ + { 'itunes:author': channelClaim.name }, + { + 'itunes:category': [ + { + _attr: { + text: getItunesCategory(channelClaim), + }, + }, + ], + }, + generateItunesImageElement(channelClaim), + generateItunesOwnerElement(channelClaim), + generateItunesExplicitElement(channelClaim), + ], + }); + + // --- Content --- + claimsInChannel.forEach((c) => { + const title = (c.value && c.value.title) || c.name; + const thumbnailUrl = (c.value && c.value.thumbnail && c.value.thumbnail.url) || ''; + const thumbnailHtml = thumbnailUrl + ? `

thumbnail

` + : ''; + const description = thumbnailHtml + getFormattedDescription(c); + + feed.item({ + title: title, + description: description, + url: `${URL}/${c.name}:${c.claim_id}`, + guid: undefined, // defaults to 'url' + author: undefined, // defaults feed author property + date: new Date(c.meta ? c.meta.creation_timestamp * 1000 : null), + enclosure: generateEnclosureForClaimContent(c), + custom_elements: [ + { 'itunes:title': title }, + { 'itunes:author': channelClaim.name }, + generateItunesImageElement(c), + generateItunesDurationElement(c), + generateItunesExplicitElement(c), + ], }); }); return feed; } -function postProcess(feed) { - // Handle 'Feed' creating an invalid MIME type when trying to guess - // from 'https://thumbnails.lbry.com/UCgQ8eREJzR1dO' style of URLs. - return feed.replace(/type="image\/\/.*"\/>/g, 'type="image/*"/>'); -} - async function getRss(ctx) { if (!ctx.params.claimName || !ctx.params.claimId) { return 'Invalid URL'; } - const channelClaim = await getChannelClaim(ctx.params.claimName, ctx.params.claimId); - if (typeof channelClaim === 'string' || !channelClaim) { - return channelClaim; + const { claim: channelClaim, error } = await getChannelClaim(ctx.params.claimName, ctx.params.claimId); + if (error) { + return error; } - const feed = await getFeed(channelClaim, `${URL}${ctx.request.url}`); - return postProcess(feed.rss2()); + const latestClaimsInChannel = await getClaimsFromChannel(channelClaim.claim_id, NUM_ENTRIES); + const feed = generateFeed(`${URL}${ctx.request.url}`, channelClaim, latestClaimsInChannel); + return feed.xml(); } module.exports = { getRss }; diff --git a/yarn.lock b/yarn.lock index b896797ba..3206d9f07 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7205,13 +7205,6 @@ fd-slicer@~1.1.0: dependencies: pend "~1.2.0" -feed@^4.2.2: - version "4.2.2" - resolved "https://registry.yarnpkg.com/feed/-/feed-4.2.2.tgz#865783ef6ed12579e2c44bbef3c9113bc4956a7e" - integrity sha512-u5/sxGfiMfZNtJ3OvQpXcvotFpYkL0n9u9mM2vkui2nGo8b4wvDkJ8gAkYqbA8QpGyFCv3RK0Z+Iv+9veCS9bQ== - dependencies: - xml-js "^1.6.11" - figgy-pudding@^3.5.1: version "3.5.2" resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.2.tgz#b4eee8148abb01dcf1d1ac34367d59e12fa61d6e" @@ -10908,6 +10901,18 @@ mime-db@1.48.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.48.0.tgz#e35b31045dd7eada3aaad537ed88a33afbef2d1d" integrity sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ== +mime-db@~1.25.0: + version "1.25.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.25.0.tgz#c18dbd7c73a5dbf6f44a024dc0d165a1e7b1c392" + integrity sha1-wY29fHOl2/b0SgJNwNFloeexw5I= + +mime-types@2.1.13: + version "2.1.13" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.13.tgz#e07aaa9c6c6b9a7ca3012c69003ad25a39e92a88" + integrity sha1-4HqqnGxrmnyjASxpADrSWjnpKog= + dependencies: + mime-db "~1.25.0" + mime-types@^2.1.12, mime-types@~2.1.19: version "2.1.31" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.31.tgz#a00d76b74317c61f9c2db2218b8e9f8e9c5c9e6b" @@ -14307,6 +14312,14 @@ roarr@^2.15.3: semver-compare "^1.0.0" sprintf-js "^1.1.2" +rss@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/rss/-/rss-1.2.2.tgz#50a1698876138133a74f9a05d2bdc8db8d27a921" + integrity sha1-UKFpiHYTgTOnT5oF0r3I240nqSE= + dependencies: + mime-types "2.1.13" + xml "1.0.1" + rtlcss@2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/rtlcss/-/rtlcss-2.5.0.tgz#455549e49113f9e1cf83169a44de526c816de8a4" @@ -17208,18 +17221,16 @@ xdg-basedir@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-4.0.0.tgz#4bc8d9984403696225ef83a1573cbbcb4e79db13" -xml-js@^1.6.11: - version "1.6.11" - resolved "https://registry.yarnpkg.com/xml-js/-/xml-js-1.6.11.tgz#927d2f6947f7f1c19a316dd8eea3614e8b18f8e9" - integrity sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g== - dependencies: - sax "^1.2.4" - xml-name-validator@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-2.0.1.tgz#4d8b8f1eccd3419aa362061becef515e1e559635" integrity sha1-TYuPHszTQZqjYgYb7O9RXh5VljU= +xml@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" + integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU= + xmlbuilder@^10.0.0: version "10.1.1" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz#8cae6688cc9b38d850b7c8d3c0a4161dcaf475b0"