diff --git a/models/Rating.js b/models/Rating.js new file mode 100644 index 0000000..6a8c7b0 --- /dev/null +++ b/models/Rating.js @@ -0,0 +1,7 @@ +function Rating(score, link, service) { + this.score = score; + this.link = link; + this.service = service; +} + +module.exports = Rating; \ No newline at end of file diff --git a/router.js b/router.js index 9e83f5f..b318af3 100644 --- a/router.js +++ b/router.js @@ -7,7 +7,7 @@ module.exports = (dir, s) => {//s is the services layer. router.get('/manual', require(dir + 'manualRoute.js')()); router.get('/cards', require(dir + 'cardsRoute.js')(s.data)); router.get('/stats', require(dir + 'statsRoute.js')(s.data)); - router.get('/retry', require(dir + 'retryRoute.js')(s.data, s.scoreUpdate)); + router.get('/reset', require(dir + 'resetRoute.js')(s.data, s.scoreUpdate)); //router.post('/updategame', require(dir + 'updateScoreForGame.js')(s.data, s.metacritic)); return router; }; \ No newline at end of file diff --git a/routes/cardsRoute.js b/routes/cardsRoute.js index 85ce588..2bf3956 100644 --- a/routes/cardsRoute.js +++ b/routes/cardsRoute.js @@ -2,9 +2,10 @@ module.exports = (dataService) => { return (req,res) => { - const ratedGames = dataService.getRatedGames().sort( (a, b) => b.score - a.score); - const unratedGames = dataService.getUnratedGames(); - const games = ratedGames.concat(unratedGames); + const ratedGames = dataService.getRatedGames().sort( (a, b) => b.rating_score - a.rating_score); + const withoutScore = dataService.getGamesWithoutScore(); + const unratedGames = dataService.getGamesWithoutRating(); + const games = ratedGames.concat(withoutScore, unratedGames); res.render('cards', { gamesList: games }); }; }; \ No newline at end of file diff --git a/routes/defaultRoute.js b/routes/defaultRoute.js index ee0cf85..273cb3d 100644 --- a/routes/defaultRoute.js +++ b/routes/defaultRoute.js @@ -4,9 +4,10 @@ const fs = require('fs'); module.exports = (dataService, fetchService) => { return (req,res) => { - const ratedGames = dataService.getRatedGames().sort( (a, b) => b.score - a.score); - const unratedGames = dataService.getUnratedGames(); - const games = ratedGames.concat(unratedGames); + const ratedGames = dataService.getRatedGames().sort( (a, b) => b.rating_score - a.rating_score); + const withoutScore = dataService.getGamesWithoutScore(); + const unratedGames = dataService.getGamesWithoutRating(); + const games = ratedGames.concat(withoutScore, unratedGames); const lastUpdate = fetchService.getLastFetchDate(); const versionInfo = Settings.buildVersion || Settings.buildVersion || 'local build'; res.render('games', { title: games.length + ' Games on Sale', gamesList: games, lastFetch: lastUpdate, version: versionInfo }); diff --git a/routes/retryRoute.js b/routes/resetRoute.js similarity index 56% rename from routes/retryRoute.js rename to routes/resetRoute.js index e73dc4b..4eefac8 100644 --- a/routes/retryRoute.js +++ b/routes/resetRoute.js @@ -3,8 +3,8 @@ module.exports = (gameService, scoreUpdateService) => { return async (req, res) => { - gameService.retry(); - scoreUpdateService.checkAndUpdateScores(); + gameService.resetRatingForGamesWithoutScore(); + await scoreUpdateService.checkAndUpdateScores(); res.sendStatus(202); }; }; \ No newline at end of file diff --git a/servicelayer.js b/servicelayer.js index a2ebade..d2d432c 100644 --- a/servicelayer.js +++ b/servicelayer.js @@ -2,10 +2,12 @@ module.exports = (dir) => { const nintendo = require(dir + '/nintendoShopService')(); const metacritic = require(dir + '/metacriticService')(); + const opencritic = require(dir + '/opencriticService')(); const data = require(dir + '/gameService')(); + const rating = require(dir + '/ratingService')(opencritic, metacritic, data); const saleHistory = require(dir + '/saleHistoryService')(); const fetch = require(dir + '/fetchService')(data, nintendo, saleHistory); - const scoreUpdate = require(dir + '/scoreUpdateService')(data, metacritic); + const scoreUpdate = require(dir + '/scoreUpdateService')(data, rating); const cronjobFetch = require(dir + '/fetchCronjob')(fetch); const cronjobScore = require(dir + '/scoreCronjob')(scoreUpdate); return { @@ -13,6 +15,8 @@ module.exports = (dir) => { saleHistory, nintendo, metacritic, + opencritic, + rating, fetch, scoreUpdate, cronjobFetch, diff --git a/services/fetchService.js b/services/fetchService.js index e5d4f2a..cbea190 100644 --- a/services/fetchService.js +++ b/services/fetchService.js @@ -9,6 +9,8 @@ function Game(game) { this.imageUrl = 'https:' + game.image_url_sq_s; this.priceRegular = game.price_regular_f; this.nintendoUrl = game.url; + this.rating_available = false; + this.rating_hasScore = false; } function Sleep(milliseconds) { @@ -42,7 +44,7 @@ module.exports = (dataService, nintendoService, saleService) => { if (gamesToAdd != undefined && gamesToAdd.length >= 0) { const priceInfos = await nintendoService.getPriceInfoForGames(gamesToAdd.map(game => game.nsId)); gamesToAddWithSaleDetails = gamesToAdd - .filter(game => priceInfos.get(game.nsId) != undefined) // remove games without price discount + .filter(game => typeof priceInfos.get(game.nsId) !== 'undefined') // remove games without price discount .map(game => { const info = priceInfos.get(game.nsId); const historyEntry = saleService.addSale(game.nsId, new Sale(info.price, info.start, info.end)); diff --git a/services/gameService.js b/services/gameService.js index dc520a2..1158c58 100644 --- a/services/gameService.js +++ b/services/gameService.js @@ -23,55 +23,67 @@ module.exports = () => { }, getRatedGames: () => { - return db.games.find().filter(g => (g.score != undefined)); + return db.games.find({rating_available: true, rating_hasScore: true}); }, - getUnratedGames: () => { - return db.games.find().filter(g => (g.score == undefined)); + // rating website link exists, but rating is not available yet + getGamesWithoutScore: () => { + return db.games.find({rating_available: true, rating_hasScore: false}); + }, + + // could not be found at any rating website + getGamesWithoutRating: () => { + return db.games.find({rating_available: false}); }, getStats: () => { - const tbd = db.games.find().filter(g => (g.score === 0)).length; - const notFound = db.games.find().filter(g => (g.score === -1)).length; - return { tbd: tbd, notFound: notFound } + const gameOnSale = db.games.count(); + const gamesWithRating = module.exports().getRatedGames().length; + const tbd = module.exports().getGamesWithoutScore().length; + const notFound = module.exports().getGamesWithoutRating().length; + return { notFound: notFound, unrated: tbd, rated: gamesWithRating, total: gameOnSale}; }, saveGame: (game) => { db.games.save(game); }, - retry: () => { - + // FIXME - resets everything. Seems like using a query with more than one parameter might be broken? + resetRatingForGamesWithoutScore: () => { const options = { multi: true - } - - const queryTba = { - score: 0 }; - - const queryNotFound = { - score: -1 + const queryWithoutScore = { + //rating_score: undefined + //not working as expected + rating_available: true, + rating_hasScore: false }; - const update = { - score: undefined - } - - db.games.update(queryTba, update, options); - db.games.update(queryNotFound, update, options); + rating_providers: undefined, + rating_available: false, + rating_hasScore: false, + rating_score: undefined + }; + db.games.update(queryWithoutScore, update, options); }, - setMetacritInfo: (id, rating, url) => { - const score = (rating === 'tbd') ? 0 : rating; + getRatingsFromProviders: (id) => { + return db.games.findOne({ _id: id }).rating_providers; + }, + setRating: (id, score, ratings) => { + const hasScore = typeof score !== 'undefined'; + const hasRating = (typeof ratings !== 'undefined' && ratings.length > 0); let query = { _id: id }; let update = { - score: score, - metacriticUrl: url - } + rating_available: hasRating, + rating_hasScore: hasScore, + rating_providers: ratings, + rating_score: score + }; db.games.update(query, update); } } diff --git a/services/metacriticScrapeService.js b/services/metacriticScrapeService.js index db132ac..1400e40 100644 --- a/services/metacriticScrapeService.js +++ b/services/metacriticScrapeService.js @@ -4,9 +4,9 @@ var request = require('request') , cheerio = require('cheerio') , extend = require('extend'); -var url = 'https://www.metacritic.com/' - , urlSearchAll = 'search/{0}/{1}/results' - , urlSearchAllByPlatform = 'search/{0}/{1}/results?search_type=advanced&plats[{2}]=1' +var url = 'https://www.metacritic.com' + , urlSearchAll = '/search/{0}/{1}/results' + , urlSearchAllByPlatform = '/search/{0}/{1}/results?search_type=advanced&plats[{2}]=1' , urlPage = '?page={0}' , currentPage = 0; diff --git a/services/metacriticService.js b/services/metacriticService.js index cbe012d..0bb58ac 100644 --- a/services/metacriticService.js +++ b/services/metacriticService.js @@ -1,17 +1,36 @@ 'use strict'; -const Settings = require('../settings'); +const Rating = require('../models/Rating'); +const service = 'metacritic'; const metacriticScrape = require('./metacriticScrapeService'); - +const { promisify } = require('util'); +const levenshtein = require('fast-levenshtein'); const platformIdSwitch = 268409; const categoryGame = 'game'; -const searchSwitchGame = async (title, cb) => { +const asyncScrapeSearch = promisify(metacriticScrape.Search); + +async function searchSwitchGame(title) { const searchTitle = getSearchTitle(title); console.log(`${title} = ${searchTitle}`); const options = { text: searchTitle, category: categoryGame, platformId: platformIdSwitch }; + return asyncScrapeSearch(options); +} + +const getBestMatchTitle = (title, titlesFromMC) => { + var lowestScore = 99; + var bestMatch = titlesFromMC[0]; - metacriticScrape.Search(options, cb); + titlesFromMC.forEach(game => { + const score = levenshtein.get(title, game.title); + if (score < lowestScore) { + lowestScore = score; + bestMatch = game; + } + }); + + console.log(`Found best match for "${title}" with a score of ${lowestScore}: "${bestMatch.title}"`); + return bestMatch; } function getSearchTitle(title) { @@ -24,8 +43,29 @@ function getSearchTitle(title) { .trim(); } +async function getRatingFor(title) { + try { + const list = await searchSwitchGame(title); + const bestMatch = getBestMatchTitle(title, list); + console.debug(`${title}: ${bestMatch.metascore}, ${bestMatch.link}`); + let score = (bestMatch.metascore !== 'tbd') ? parseInt(bestMatch.metascore) : undefined; + return new Rating(score, bestMatch.link, service); + } + catch (err) { + console.error(`failed to fetch score for "${title}": ${err}`); + if (err === 'No results') { + return new Rating(undefined, undefined, service); + } + else { + // error but no results... what should we do here? + return new Rating(undefined, undefined, service); + } + } +} + module.exports = () => { return { - searchSwitchGame + searchSwitchGame: searchSwitchGame, + getRatingFor: getRatingFor }; } \ No newline at end of file diff --git a/services/opencriticService.js b/services/opencriticService.js new file mode 100644 index 0000000..9b3d3dc --- /dev/null +++ b/services/opencriticService.js @@ -0,0 +1,67 @@ +'use strict'; +const axios = require('axios').default; +const Rating = require('../models/Rating'); +const Settings = require('../settings'); +const service = 'opencritic'; + +async function searchGame(title) { + try { + const titleEncoded = encodeURIComponent(title); + const response = await axios({ + method: 'get', + url: Settings.opencriticBase + '/meta/search', + params: { + criteria: titleEncoded + }, + }); + + let first = response.data[0]; + // check dist and discard if bigger than 0.59 + if (first.dist <= 0.59) { + return {error: false, id: first.id, name: first.name}; + } + console.log('[info] opencritic - ' + title + ' - not found - distance too high'); + return {error: false, id: undefined, name: undefined}; + } + catch (e) { + console.log('[error] opencritic - ' + title + ' - search request failed - criteria: ' + titleEncoded); + throw Error('opencritic - ' + title + ' - search request failed - criteria: ' + titleEncoded); + } +} + +async function getScore(id) { + try { + const response = await axios({ + method: 'get', + url: Settings.opencriticBase + '/game/' + id, + }); + let score = (response.data.medianScore > -1) ? response.data.medianScore : undefined; + return {error: false, score: score}; + } + catch (e) { + console.log('[error] opencritic - score request failed - id: ' + id); + throw Error('opencritic - score request failed - id: ' + id); + } +} + +async function getRatingFor(title) { + let link = undefined; + try { + const searchResult = await searchGame(title); + if (typeof searchResult.id === 'undefined') { + return new Rating(undefined, undefined, service); + } + link = 'https://opencritic.com/game/' + searchResult.id + '/' + searchResult.name.replace(/ /g, "-"); + let rating = await getScore(searchResult.id); + return new Rating(rating.score, link, service); + } + catch (e) { + return new Rating(undefined, link, service); + } +} + +module.exports = () => { + return { + getRatingFor: getRatingFor + } +}; \ No newline at end of file diff --git a/services/ratingService.js b/services/ratingService.js new file mode 100644 index 0000000..3615297 --- /dev/null +++ b/services/ratingService.js @@ -0,0 +1,87 @@ +'use strict'; + +/** + * + * @param providerQueries + * @returns average Score and ratings obtained from game rating providers + * + * Tries to query all rating providers and calculates the average score from all obtained ratings. But it might + * occur that the game is listed at the given providers, but hasn't got any score yet. In such cases the returned + * average score will be set to '-1'. + */ +async function getRatingFromProviders(providerQueries) { + try { + let ratings = []; + let avgScore = -1; + + const results = await Promise.all(providerQueries); + results.map((rating) => { + if (typeof rating.link !== 'undefined') { + ratings.push(rating); + } + }); + + let scores = ratings.filter( x => typeof x.score !== 'undefined' ); + if (scores.length > 0) { + let sum = scores.reduce((previous, current) => ({ score: previous.score + current.score })); + avgScore = sum.score / scores.length; + } + + return {avgScore: avgScore, ratings: ratings}; + } + catch (err) { + console.log(err); + } +} + +function getRatingUpdate(ratingsFromRepo, ratingFromProvider) { + let ratingsToReturn = []; + let avgScore = undefined; + + // add rating from provider or replace existing rating + if (typeof ratingsFromRepo !== 'undefined') { + ratingsToReturn = ratingsFromRepo.filter(entry => entry.service !== ratingFromProvider.service) + } + if (typeof ratingFromProvider.link !== 'undefined') { + // add rating + ratingsToReturn.push(ratingFromProvider); + } + else { + // game wasn't found + } + + // update average score + let scores = ratingsToReturn.filter( x => typeof x.score !== 'undefined' ); + if (scores.length > 0) { + let sum = scores.reduce((previous, current) => ({ score: previous.score + current.score })); + avgScore = Math.round(sum.score / scores.length); + } + + return {score: avgScore, ratings: ratingsToReturn}; +} + +module.exports = (opencritic, metacritic, gameRepo) => { + return { + updateRatingOf: async (game) => { + opencritic.getRatingFor(game.title) + .then( (rating) => { + const ratingsFromRepo = gameRepo.getRatingsFromProviders(game._id); + const update = getRatingUpdate(ratingsFromRepo, rating); + gameRepo.setRating(game._id, update.score, update.ratings); + }); + metacritic.getRatingFor(game.title) + .then((rating) => { + const ratingsFromRepo = gameRepo.getRatingsFromProviders(game._id); + const update = getRatingUpdate(ratingsFromRepo, rating); + gameRepo.setRating(game._id, update.score, update.ratings); + }); + // let queryOC = opencritic.getRatingFor(game.title); + //let queryMC = metacritic.getRatingFor(game.title); + // const rating = await getRatingFromProviders([queryOC, queryMC]); + // update rating only if any rating (with or without score) was returned from the given rating providers + // if (typeof rating.ratings !== 'undefined' && rating.ratings.length) { + // gameRepo.setRating(game._id, rating); + // } + } + } +}; \ No newline at end of file diff --git a/services/scoreCronjob.js b/services/scoreCronjob.js index 307bb8f..391da05 100644 --- a/services/scoreCronjob.js +++ b/services/scoreCronjob.js @@ -5,6 +5,6 @@ const ScoreCronjob = require('cron').CronJob; module.exports = async (scoreUpdateService) => { return new ScoreCronjob('0 */5 * * * *', async function () { console.log('cron job started - score update using metacritic'); - scoreUpdateService.checkAndUpdateScores(); + await scoreUpdateService.checkAndUpdateScores(); }, null, true, ''); }; \ No newline at end of file diff --git a/services/scoreUpdateService.js b/services/scoreUpdateService.js index 3c66b92..0733d7b 100644 --- a/services/scoreUpdateService.js +++ b/services/scoreUpdateService.js @@ -1,51 +1,13 @@ 'use strict'; -const levenshtein = require('fast-levenshtein'); -const getBestMatchTitle = (title, titlesFromMC) => { - var lowestScore = 99; - var bestMatch = titlesFromMC[0]; - - titlesFromMC.forEach(game => { - const score = levenshtein.get(title, game.title); - if (score < lowestScore) { - lowestScore = score; - bestMatch = game; - } - }); - - console.log(`Found best match for "${title}" with a score of ${lowestScore}: "${bestMatch.title}"`); - return bestMatch; -} - -module.exports = (dataService, metacriticService) => { +module.exports = (dataService, ratingService) => { return { lastUpdate: undefined, - checkAndUpdateScores: () => { - let games = dataService.getUnratedGames(); - - var i = 0; - - games.slice(0, 100).map(game => { - metacriticService.searchSwitchGame(game.title, (err, list) => { - if (err) { - console.error(`failed to fetch score for "${game.title}": ${err}`); - if (err === 'No results') { - dataService.setMetacritInfo(game._id, -1, null); - } - } else if (list && list[0]) { - const bestMatch = getBestMatchTitle(game.title, list); - - console.debug(`${game.title}: ${bestMatch.metascore}, ${bestMatch.link}`); - dataService.setMetacritInfo(game._id, bestMatch.metascore, bestMatch.link); - } - }); + checkAndUpdateScores: async () => { + let games = dataService.getGamesWithoutRating(); + games.slice(0,100).map(async game => { + await ratingService.updateRatingOf(game); }) - }, - getLastUpdateDate: () => { - if (this.lastUpdate != null) - return Date(this.lastUpdate).toString(); - else - return 'not updated yet' } } }; \ No newline at end of file diff --git a/settings.js b/settings.js index dc59fde..b69c72a 100644 --- a/settings.js +++ b/settings.js @@ -2,5 +2,6 @@ require('dotenv').config({ silent: true }); module.exports = { port: process.env.PORT || 3000, + opencriticBase: process.env.API_BASE_OPENCRITIC || 'https://api.opencritic.com/api', buildVersion: process.env.IMAGE_VERSION || undefined }; \ No newline at end of file diff --git a/views/cards.pug b/views/cards.pug index efd31bd..2169a86 100644 --- a/views/cards.pug +++ b/views/cards.pug @@ -8,7 +8,12 @@ mixin game(item) .switcharoo-card_priceContainer(style='flex-grow: 1') mdc-typography.mdc-typography--headline6(style='margin-right: .31rem; opacity: .3; text-decoration: line-through') €#{item.priceRegular} span.mdc-typography.mdc-typography--headline6 €#{item.saleDetails.price} - .mdc-typography.mdc-typography--headline6 #{item.score} + if item.rating_hasScore + .mdc-typography.mdc-typography--headline6 #{item.rating_score} + else if item.rating_available + .mdc-typography.mdc-typography--headline6 tbd + else + .mdc-typography.mdc-typography--headline6 ? .mdc-card__media.mdc-card__media--square(style='background-image: url(' + item.imageUrl + ')') .switcharoo-card__info(style='padding: 8px;') .mdc-typography.mdc-typography--headline6(style="font-size: .88rem;") #{item.title} @@ -17,8 +22,8 @@ mixin game(item) .mdc-card__action-icons a(href='https://www.nintendo.de' + item.nintendoUrl rel="noopener noreferrer" target="_blank") button.material-icons.mdc-icon-button.mdc-card__action.mdc-card__action--icon(title='Nintendo Store') storefront - if item.score != null - a(href=item.metacriticUrl rel="noopener noreferrer" target="_blank") + if item.rating_available + a(href=item.rating_providers[0].link rel="noopener noreferrer" target="_blank") button.material-icons.mdc-icon-button.mdc-card__action.mdc-card__action--icon(title='Metacritic') show_chart block content diff --git a/views/games.pug b/views/games.pug index 4a28da2..a4ca434 100644 --- a/views/games.pug +++ b/views/games.pug @@ -2,10 +2,12 @@ extends layout mixin game(item) tr(class="mdc-data-table__row").content - if item.score == null + if !item.rating_available td - + else if !item.rating_hasScore + td tbd else - td.metascore #{item.score} + td.metascore #{item.rating_score} td #{item.discount} td #{item.title} td #{item.saleDetails.price} @@ -14,8 +16,8 @@ mixin game(item) td a(class="mdc-icon-button meterial-icons" href='https://www.nintendo.de' + item.nintendoUrl rel="noopener noreferrer" target="_blank") i(class="material-icons mdc-button__icon") storefront - if item.score != null && item.score > -1 - a(class="mdc-icon-button meterial-icons" href=item.metacriticUrl rel="noopener noreferrer" target="_blank") + if rating_available + a(class="mdc-icon-button meterial-icons" href=item.rating_providers[0].link rel="noopener noreferrer" target="_blank") i(class="material-icons mdc-button__icon") show_chart block content