diff --git a/bot/modules/claimbot.js b/bot/modules/claimbot.js index 68d14e2..2a8666a 100644 --- a/bot/modules/claimbot.js +++ b/bot/modules/claimbot.js @@ -1,259 +1,201 @@ -'use strict'; +"use strict"; let lbry; let mongo; let discordBot; -let moment = require('moment'); -let request = require('request'); -let sleep = require('sleep'); -let config = require('config'); -let channels = config.get('claimbot').channels; -const Discord = require('discord.js'); - +let moment = require("moment"); +let request = require("request"); +let sleep = require("sleep"); +let config = require("config"); +let channels = config.get("claimbot").channels; +const Discord = require("discord.js"); +const rp = require("request-promise"); +const jsonfile = require("jsonfile"); +const path = require("path"); +const fs = require("fs"); +const appRoot = require("app-root-path"); +const fileExists = require("file-exists"); module.exports = { init: init }; function init(discordBot_) { if (lbry) { - throw new Error('init was already called once'); + throw new Error("init was already called once"); } discordBot = discordBot_; - const MongoClient = require('mongodb').MongoClient; - MongoClient.connect(config.get('mongodb').url, function(err, db) { + const MongoClient = require("mongodb").MongoClient; + MongoClient.connect(config.get("mongodb").url, function(err, db) { if (err) { throw err; } mongo = db; - const bitcoin = require('bitcoin'); - lbry = new bitcoin.Client(config.get('lbrycrd')); - - console.log('Activating claimbot '); - discordBot.channels.get(channels[0]).send('activating claimbot'); + console.log("Activating claimbot "); + discordBot.channels.get(channels[0]).send("activating claimbot"); + // Check that our syncState file exist. + fileExists(path.join(appRoot.path, "syncState.json"), (err, exists) => { + if (err) { + throw err; + } + if (!exists) { + fs.writeFileSync(path.join(appRoot.path, "syncState.json"), "{}"); + } + }); setInterval(function() { - announceNewClaims(); + announceClaims(); }, 60 * 1000); - announceNewClaims(); + announceClaims(); }); } -function announceNewClaims() { - if (!mongo) { - discordPost('Failed to connect to mongo', {}); - return; +async function announceClaims() { + // get last block form the explorer API. + let lastBlockHeight = JSON.parse( + await rp("https://explorer.lbry.io/api/v1/status") + ).status.height; + // get the latest claims from chainquery since last sync + let syncState = await getJSON(path.join(appRoot.path, "syncState.json")); // get our persisted state + if (!syncState.LastSyncTime) { + syncState.LastSyncTime = new Date() + .toISOString() + .slice(0, 19) + .replace("T", " "); } - - if (!lbry) { - discordPost('Failed to connect to lbrycrd', {}); - return; - } - - Promise.all([getLastBlock(), lbryCall('getinfo')]) - .then(function([lastProcessedBlock, currentBlockInfo]) { - const currentHeight = currentBlockInfo['blocks']; - console.log(currentHeight); - if (lastProcessedBlock === null) { - console.log('First run. Setting last processed block to ' + currentHeight + ' and exiting.'); - return setLastBlock(currentHeight); - } - - const testBlock = false; - - if (testBlock || lastProcessedBlock < currentHeight) { - const firstBlockToProcess = testBlock || lastProcessedBlock + 1, - lastBlockToProcess = testBlock || currentHeight; - - console.log('Doing blocks ' + firstBlockToProcess + ' to ' + lastBlockToProcess); - return announceClaimsLoop(firstBlockToProcess, lastBlockToProcess, currentHeight); - } - }) - .catch(function(err) { - discordPost(err.stack, {}); - }); -} - -function announceClaimsLoop(block, lastBlock, currentHeight) { - let claimsFound = 0; - return lbryCall('getblockhash', block) - .then(function(blockHash) { - return lbryCall('getblock', blockHash); - }) - .then(function(blockData) { - return Promise.all(blockData['tx'].map(getClaimsForTxid)); - }) - .then(function(arrayOfClaimArrays) { - const claims = Array.prototype.concat(...arrayOfClaimArrays).filter(function(c) { - return !!c; - }); - console.log('Found ' + claims.length + ' claims in ' + block); - claimsFound = claims.length; - return Promise.all( - claims.map(function(claim) { - //the API has a rate limit. to avoid hitting it we must have a small delay between each message - //if claims were found in this block, then we wait, otherwise we don't - if (claimsFound > 0 && claim.hasOwnProperty('claimId')) sleep.msleep(300); - return announceClaim(claim, block, currentHeight); - }) - ); - }) - .then(function() { - return setLastBlock(block); - }) - .then(function() { - const nextBlock = block + 1; - if (nextBlock <= lastBlock) { - return announceClaimsLoop(nextBlock, lastBlock, currentHeight); - } - }); -} - -function announceClaim(claim, claimBlockHeight, currentHeight) { - console.log('' + claimBlockHeight + ': New claim for ' + claim['name']); - console.log(claim); - - //ignore supports for now - //the issue with supports is that they should be treated completely differently - //they are not new claims... - if (claim.hasOwnProperty('supported claimId')) return; - - let options = { - method: 'GET', - url: 'http://127.0.0.1:5000/claim_decode/' + claim['name'] - }; - - request(options, function(error, response, body) { - if (error) throw new Error(error); - try { - console.log(JSON.stringify(JSON.parse(body), null, 2)); - let claimData = null; - let channelName = null; - try { - body = JSON.parse(body); - if (body.hasOwnProperty('stream') && body.stream.hasOwnProperty('metadata')) { - claimData = body.stream.metadata; - channelName = body.hasOwnProperty('channel_name') ? body['channel_name'] : null; - } - } catch (e) { - console.error(e); - return; - } - - return Promise.all([lbryCall('getvalueforname', claim['name']), lbryCall('getclaimsforname', claim['name'])]).then(function([currentWinningClaim, claimsForName]) { - //console.log(JSON.stringify(claimData)); - let value = null; - if (claimData !== null) value = claimData; - else { - try { - value = JSON.parse(claim['value']); - } catch (e) {} - } - - const text = []; - - if (value) { - /* - if (channelName) { - text.push("Channel: lbry://" + channelName); - } - else - */ - console.log(value); - if (value['author']) { - text.push('author: ' + value['author']); - } - if (value['description']) { - text.push(value['description']); - } - // if (value['content_type']) - // { - // text.push("*Content Type:* " + value['content_type']); - // } - if (value['nsfw']) { - text.push('*Warning: Adult Content*'); - } - - //"fee":{"currency":"LBC","amount":186,"version":"_0_0_1","address":"bTGoFCakvQXvBrJg1b7FJzombFUu6iRJsk"} - if (value['fee']) { - const fees = []; - text.push('Price: ' + value['fee'].amount + ' *' + value['fee'].currency + '*'); - } - - if (!claim['is controlling']) { - // the following is based on https://lbry.io/faq/claimtrie-implementation - const lastTakeoverHeight = claimsForName['nLastTakeoverHeight'], - maxDelay = 4032, // 7 days of blocks at 2.5min per block - activationDelay = Math.min(maxDelay, Math.floor((claimBlockHeight - lastTakeoverHeight) / 32)), - takeoverHeight = claimBlockHeight + activationDelay, - secondsPerBlock = 161, // in theory this should be 150, but in practice its closer to 161 - takeoverTime = Date.now() + (takeoverHeight - currentHeight) * secondsPerBlock * 1000; - - text.push('Takes effect on approx. **' + moment(takeoverTime, 'x').format('MMMM Do [at] HH:mm [UTC]') + '** (block ' + takeoverHeight + ')'); - } - - const richEmbeded = { - author: { - name: value['author'] || 'Anonymous', - url: `http://open.lbry.io/${claim['name']}#${claim['claimId']}`, - icon_url: 'http://barkpost-assets.s3.amazonaws.com/wp-content/uploads/2013/11/3dDoge.gif' - }, - title: 'lbry://' + (channelName ? channelName + '/' : '') + claim['name'], - color: 1399626, - description: escapeSlackHtml(text.join('\n')), - footer: { - text: 'Block ' + claimBlockHeight + ' • Claim ID ' + claim['claimId'] - }, - image: { url: !value['nsfw'] ? value['thumbnail'] || '' : '' }, - url: `http://open.lbry.io/${claim['name']}#${claim['claimId']}` - }; - - discordPost(text, richEmbeded); - } - }); - } catch (e) { - console.error(e); + let claimsSince = JSON.parse(await getClaimsSince(syncState.LastSyncTime)) + .data; + // filter out the claims that we should add to discord + let claims = []; + for (let claim of claimsSince) { + claim.value = JSON.parse(claim.value); + if (claim.value.Claim && claim.value.Claim.stream) { + claim.metadata = claim.value.Claim.stream.metadata; + } else { + claim.metadata = null; } - }); + if (claim.bid_state !== "Spent" || claim.bid_state !== "Expired") { + claims.push(claim); + } + } + for (let claim of claims) { + console.log(claim); + } + // send each claim to discord. + for (let claim of claims) { + console.log(claim); + if (claim.metadata) { + // If its a claim, make a claimEmbed + let claimEmbed = new Discord.RichEmbed() + .setAuthor( + claim.channel + ? `New claim from ${claim.channel}` + : "New claim from Anonymous", + "http://barkpost-assets.s3.amazonaws.com/wp-content/uploads/2013/11/3dDoge.gif", + `http://open.lbry.io/${ + claim.channel + ? `${claim.channel}#${claim.channelId}/${claim["name"]}` + : `${claim["name"]}#${claim["claimId"]}` + }` + ) + .setTitle( + "lbry://" + (claim.channel ? `${claim.channel}/` : "") + claim["name"] + ) + .setURL( + `http://open.lbry.io/${ + claim.channel + ? `${claim.channel}#${claim.channelId}/${claim["name"]}` + : `${claim["name"]}#${claim["claimId"]}` + }` + ) + .setColor(1399626) + .setFooter( + `Block ${claim.height} • Claim ID ${ + claim.claimId + } • Data from Chainquery` + ); + if (claim.metadata["title"]) + claimEmbed.addField("Title", claim.metadata["title"]); + if (claim.metadata["author"]) + claimEmbed.addField("Author", claim.metadata["author"]); + if (claim.metadata["description"]) { + claimEmbed.addField( + "Description", + claim.metadata["description"].substring(0, 1020) + ); + } + if (claim.metadata["fee"]) + claimEmbed.addField( + "Fee", + claim.metadata["fee"].amount + " " + claim.metadata["fee"].currency + ); + if (claim.metadata["license"] && claim.metadata["license"].length > 2) + claimEmbed.addField("License", claim.metadata["license"]); + if (!claim.metadata["nsfw"] && claim.metadata["thumbnail"]) + claimEmbed.setThumbnail(claim.metadata["thumbnail"]); + if ( + claim.bid_state !== "Controlling" && + claim.height < claim.valid_at_height + ) { + // Claim have not taken over the old claim, send approx time to event. + let takeoverTime = + Date.now() + (claim.valid_at_height - lastBlockHeight) * 161 * 1000; // in theory this should be 150, but in practice its closer to 161 + claimEmbed.addField( + "Takes effect on approx", + moment(takeoverTime, "x").format("MMMM Do [at] HH:mm [UTC]") + + ` • at block height ${claim.valid_at_height}` + ); + } + /*claimEmbed.addField("Claimed for", `${claim.effective_amount} LBC`);*/ + discordPost(claimEmbed); + } else if (claim.name.charAt(0) === "@") { + // This is a channel claim + let channelEmbed = new Discord.RichEmbed() + .setAuthor( + "New channel claim", + "http://barkpost-assets.s3.amazonaws.com/wp-content/uploads/2013/11/3dDoge.gif", + `http://open.lbry.io/${claim["name"]}#${claim["claimId"]}` + ) + .setTitle( + "lbry://" + (claim.channel ? claim.channel + "/" : "") + claim["name"] + ) + .setURL(`http://open.lbry.io/${claim["name"]}#${claim["claimId"]}`) + .setColor(1399626) + .setFooter( + `Block ${claim.height} • Claim ID ${ + claim.claimId + } • Data from Chainquery` + ) + .addField("Channel Name", claim["name"]); + discordPost(channelEmbed); + } + } + // set the last sync time to the db. + syncState.LastSyncTime = new Date() + .toISOString() + .slice(0, 19) + .replace("T", " "); + await saveJSON(path.join(appRoot.path, "syncState.json"), syncState); } -function escapeSlackHtml(txt) { - return txt - .replace('&', '&') - .replace('<', '<') - .replace('>', '>'); -} - -function getClaimsForTxid(txid) { - return lbryCall('getclaimsfortx', txid).catch(function(err) { - // an error here most likely means the transaction is spent, - // which also means there are no claims worth looking at - return []; - }); -} - -function getLastBlock() { - return new Promise(function(resolve, reject) { - mongo.collection('claimbot').findOne({}, function(err, obj) { +function getJSON(path) { + return new Promise((resolve, reject) => { + jsonfile.readFile(path, function(err, jsoncontent) { if (err) { reject(err); - } else if (!obj) { - mongo.collection('claimbot').createIndex({ last_block: 1 }, { unique: true }); - resolve(null); } else { - resolve(obj.last_block); + resolve(jsoncontent); } }); }); } - -function setLastBlock(block) { - return new Promise(function(resolve, reject) { - mongo.collection('claimbot').findOneAndUpdate({ last_block: { $exists: true } }, { last_block: block }, { upsert: true, returnOriginal: false }, function(err, obj) { - if (!err && obj && obj.value.last_block != block) { - reject('Last value should be ' + block + ', but it is ' + obj.value.last_block); +function saveJSON(path, obj) { + return new Promise((resolve, reject) => { + jsonfile.writeFile(path, obj, function(err, jsoncontent) { + if (err) { + reject(err); } else { resolve(); } @@ -261,25 +203,45 @@ function setLastBlock(block) { }); } -function discordPost(text, params) { - let richEmbeded = new Discord.RichEmbed(params); - +function discordPost(embed) { channels.forEach(channel => { discordBot.channels .get(channel) - .send('', richEmbeded) + .send("", embed) .catch(console.error); }); } -function lbryCall(...args) { - return new Promise(function(resolve, reject) { - lbry.cmd(...args, function(err, ...response) { - if (err) { - reject(new Error('JSONRPC call failed. Args: [' + args.join(', ') + ']')); - } else { - resolve(...response); - } - }); +function getClaimsSince(time) { + return new Promise((resolve, reject) => { + let query = + `` + + `SELECT ` + + `c.name,` + + `c.valid_at_height,` + + `c.height,` + + `p.name as channel,` + + `c.publisher_id as channelId,` + + `c.bid_state,` + + `c.effective_amount,` + + `c.claim_id as claimId,` + + `c.value_as_json as value ` + + // `,transaction_by_hash_id, ` + // txhash and vout needed to leverage old format for comparison. + // `vout ` + + `FROM claim c ` + + `LEFT JOIN claim p on p.claim_id = c.publisher_id ` + + `WHERE c.created_at >='` + + time + + `'`; + // Outputs full query to console for copy/paste into chainquery (debugging) + // console.log(query); + rp(`https://chainquery.lbry.io/api/sql?query=` + query) + .then(function(htmlString) { + resolve(htmlString); + }) + .catch(function(err) { + console.log("error", "[Importer] Error getting updated claims. " + err); + reject(err); + }); }); } diff --git a/package.json b/package.json index 1dd8c1c..804cf7b 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,6 @@ { "dependencies": { + "app-root-path": "^2.1.0", "babel-cli": "^6.26.0", "babel-preset-node8": "^1.2.0", "bitcoin": "^3.0.1", @@ -8,6 +9,8 @@ "discord.js": "^11.3.2", "elmadev-discord-irc": "^2.4.1", "embed-creator": "^1.2.3", + "file-exists": "^5.0.1", + "jsonfile": "^4.0.0", "jsonpath": "^1.0.0", "moment": "^2.21.0", "mongoose": "^4.13.12", @@ -15,6 +18,7 @@ "node-config": "^0.0.2", "numeral": "^2.0.6", "request": "^2.85.0", + "request-promise": "^4.2.2", "sleep": "^5.1.1", "wget": "^0.0.1" }, diff --git a/yarn.lock b/yarn.lock index 6812c1f..5f8087e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -45,6 +45,10 @@ anymatch@^1.3.0: micromatch "^2.1.5" normalize-path "^2.0.0" +app-root-path@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-2.1.0.tgz#98bf6599327ecea199309866e8140368fd2e646a" + aproba@^1.0.3: version "1.2.0" resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" @@ -432,6 +436,10 @@ bluebird@3.5.0: version "3.5.0" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.0.tgz#791420d7f551eea2897453a8a77653f96606d67c" +bluebird@^3.5.0: + version "3.5.1" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9" + boom@2.x.x: version "2.10.1" resolved "https://registry.yarnpkg.com/boom/-/boom-2.10.1.tgz#39c8918ceff5799f83f9492a848f625add0c766f" @@ -772,6 +780,10 @@ fast-levenshtein@~2.0.4: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" +file-exists@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/file-exists/-/file-exists-5.0.1.tgz#1dcd017f787fc7be7a09a6ef3e6a3550cea31198" + filename-regex@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/filename-regex/-/filename-regex-2.0.1.tgz#c1c4b9bee3e09725ddb106b75c1e301fe2f18b26" @@ -899,7 +911,7 @@ globals@^9.18.0: version "9.18.0" resolved "https://registry.yarnpkg.com/globals/-/globals-9.18.0.tgz#aa3896b3e69b487f17e31ed2143d69a8e30c2d8a" -graceful-fs@^4.1.2, graceful-fs@^4.1.4: +graceful-fs@^4.1.2, graceful-fs@^4.1.4, graceful-fs@^4.1.6: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -1189,6 +1201,12 @@ json5@^0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/json5/-/json5-0.5.1.tgz#1eade7acc012034ad84e2396767ead9fa5495821" +jsonfile@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb" + optionalDependencies: + graceful-fs "^4.1.6" + jsonify@~0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" @@ -1246,6 +1264,10 @@ lodash.some@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/lodash.some/-/lodash.some-4.6.0.tgz#1bb9f314ef6b8baded13b549169b2a945eb68e4d" +lodash@^4.13.1: + version "4.17.10" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.10.tgz#1b7793cf7259ea38fb3661d4d38b3260af8ae4e7" + lodash@^4.14.0, lodash@^4.17.4: version "4.17.4" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" @@ -1575,6 +1597,10 @@ process-nextick-args@~1.0.6: version "1.0.7" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3" +psl@^1.1.24: + version "1.1.29" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.29.tgz#60f580d360170bb722a797cc704411e6da850c67" + punycode@^1.4.1: version "1.4.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" @@ -1672,6 +1698,21 @@ repeating@^2.0.0: dependencies: is-finite "^1.0.0" +request-promise-core@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.1.tgz#3eee00b2c5aa83239cfb04c5700da36f81cd08b6" + dependencies: + lodash "^4.13.1" + +request-promise@^4.2.2: + version "4.2.2" + resolved "https://registry.yarnpkg.com/request-promise/-/request-promise-4.2.2.tgz#d1ea46d654a6ee4f8ee6a4fea1018c22911904b4" + dependencies: + bluebird "^3.5.0" + request-promise-core "1.1.1" + stealthy-require "^1.1.0" + tough-cookie ">=2.3.3" + request@2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" @@ -1839,6 +1880,10 @@ static-eval@2.0.0: dependencies: escodegen "^1.8.1" +stealthy-require@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b" + string-width@^1.0.1, string-width@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" @@ -1913,6 +1958,13 @@ to-fast-properties@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-1.0.3.tgz#b83571fa4d8c25b82e231b06e3a3055de4ca1a47" +tough-cookie@>=2.3.3: + version "2.4.3" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781" + dependencies: + psl "^1.1.24" + punycode "^1.4.1" + tough-cookie@~2.3.0, tough-cookie@~2.3.3: version "2.3.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561"