From 44b751e58e49dff1fec690bc5fdbfcbe5e298c76 Mon Sep 17 00:00:00 2001 From: Kiril Kartunov Date: Mon, 25 Nov 2019 12:49:09 +0200 Subject: [PATCH 1/4] Implements #3461 --- config/default.js | 4 +- src/server/index.js | 4 ++ src/server/services/contentful.js | 109 ++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 1 deletion(-) diff --git a/config/default.js b/config/default.js index c392f2a8c2..4b991f9cc4 100644 --- a/config/default.js +++ b/config/default.js @@ -150,6 +150,7 @@ module.exports = { COMMUNITY_API: 'http://localhost:8000', COMMUNITY_APP_GITHUB_ISSUES: 'https://github.com/topcoder-platform/community-app/issues', EMAIL_VERIFY_URL: 'http://www.topcoder-dev.com/settings/account/changeEmail', + THRIVE_POLL_FEED: 'https://www.topcoder.com/feed', }, /* Information about Topcoder user groups can be cached in various places. @@ -259,10 +260,11 @@ module.exports = { title: 'Switch to BUSINESS', href: 'https://connect.topcoder-dev.com', }, - // Config for TC EDU + // Config for TC EDU - THRIVE TC_EDU_BASE_PATH: '/thrive', TC_EDU_TRACKS_PATH: '/tracks', TC_EDU_ARTICLES_PATH: '/articles', TC_EDU_SEARCH_PATH: '/search', TC_EDU_SEARCH_BAR_MAX_RESULTS_EACH_GROUP: 3, + TC_EDU_POLL_TIME: 3600, // 1h in seconds }; diff --git a/src/server/index.js b/src/server/index.js index a631ec428f..2e5cde2eb2 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -26,6 +26,7 @@ import { toJson as xmlToJson } from 'utils/xml2json'; import cdnRouter from './routes/cdn'; import mailChimpRouter from './routes/mailchimp'; import mockDocuSignFactory from './__mocks__/docu-sign-mock'; +import { pollArticlesForThrive } from 'server/services/contentful'; /* Dome API for topcoder communities */ import tcCommunitiesDemoApi from './tc-communities'; @@ -33,6 +34,9 @@ import tcCommunitiesDemoApi from './tc-communities'; import webpackConfigFactory from '../../webpack.config'; /* eslint-enable */ +// Init TC Blog -> THRIVE poll bridge +pollArticlesForThrive(); + global.atob = atob; const CMS_BASE_URL = `https://app.contentful.com/spaces/${config.SECRET.CONTENTFUL.SPACE_ID}`; diff --git a/src/server/services/contentful.js b/src/server/services/contentful.js index 041d9792d6..d40c37949d 100644 --- a/src/server/services/contentful.js +++ b/src/server/services/contentful.js @@ -10,8 +10,10 @@ import fetch from 'isomorphic-fetch'; // import logger from 'utils/logger'; // import moment from 'moment'; import qs from 'qs'; +import { logger } from 'topcoder-react-lib'; const contentful = require('contentful-management'); +const xml2json = require('xml2json'); /* Holds Contentful CDN URL. */ const CDN_URL = 'https://cdn.contentful.com/spaces'; @@ -191,6 +193,113 @@ export function articleVote(body) { .then(entry => entry.publish()); } +/** + * This function fetches TC RSS feed to create draft articles in THRIVE for the new posted blogs. + * It calls itself by interval to poll for new blogs. + * Runs on server side only. + */ +export async function pollArticlesForThrive() { + // make this function execute only when in production + if (process.env.BABEL_ENV !== 'production') return; + logger.log('polling blog articles for THRIVE -> INIT'); + // fetch the RSS feed and parse it + const feedXML = await fetch(config.URL.THRIVE_POLL_FEED); + if (feedXML.ok) { + let feed = await feedXML.text(); + feed = await Promise.resolve(xml2json.toJson(feed, { object: true })); + if (feed) { + // when feed loaded and parsed ok + // connect to Thrive space + const client = contentful.createClient({ + accessToken: config.SECRET.CONTENTFUL.MANAGEMENT_TOKEN, + }); + client.getSpace(config.SECRET.CONTENTFUL.EDU.SPACE_ID) + .then(space => space.getEnvironment('master')) + // loop all feed items and check if those exists in space + // if not existing then create + .then(env => Promise.all( + _.map(feed.rss.channel.item, + blogPost => env.getEntries({ + content_type: 'article', + 'fields.title[match]': blogPost.title, + }) + .then((queryData) => { + if (queryData.total === 0) { + // article not found in Contentful space + // will create it with payload... + const article = { + fields: { + title: { 'en-US': blogPost.title }, + type: { 'en-US': 'Article' }, + tags: { 'en-US': blogPost.category }, + creationDate: { 'en-US': new Date(blogPost.pubDate) }, + content: { 'en-US': blogPost.description }, + externalArticle: { 'en-US': true }, + contentUrl: { 'en-US': blogPost.link }, + }, + }; + // check if author exists + // if yes link it if no create it? + return env.getEntries({ + content_type: 'person', + query: blogPost['dc:creator'], + }) + .then((author) => { + // found an author that matches link it + if (author.total) { + article.fields.contentAuthor = { + 'en-US': [{ + sys: { + type: 'Link', linkType: 'Entry', id: author.items[0].sys.id, + }, + }], + }; + } + // try get the page to extract its featured image + return fetch(blogPost.link) + .then(rsp => rsp.text()) + .then((HTMLsrc) => { + const match = /]*href="([^"]+)"/gm.exec(HTMLsrc); + if (match && match[1]) { + // create the asset in Contentful + return env.createAsset({ + fields: { + title: { 'en-US': blogPost.title }, + file: { + 'en-US': { fileName: blogPost.title, contentType: 'image', upload: match[1] }, + }, + }, + }) + .then(asset => asset.processForAllLocales()) + .then((asset) => { + article.fields.featuredImage = { + 'en-US': { + sys: { + type: 'Link', linkType: 'Asset', id: asset.sys.id, + }, + }, + }; + return env.createEntry('article', article); + }); + } + // could not find image + // just create the article without it... + return env.createEntry('article', article); + }); + }); + } + // Article exists in Contentful space + // do nothing... + return 1; + })), + )) + .then(() => logger.log('polling blog articles for THRIVE -> DONE')); + } + } + // schedule a repeat each TC_EDU_POLL_TIME + setTimeout(pollArticlesForThrive, config.TC_EDU_POLL_TIME * 1000).unref(); +} + // /* Contentful CDN service. */ // export const cdnService = new ApiService(CDN_URL, CDN_KEY); From 1610c9d4b2f5e824660f4c3c30f939b2681e492a Mon Sep 17 00:00:00 2001 From: Kiril Kartunov Date: Mon, 25 Nov 2019 18:39:52 +0200 Subject: [PATCH 2/4] Randomize poll time for Thrive --- src/server/services/contentful.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/server/services/contentful.js b/src/server/services/contentful.js index d40c37949d..2681451ddc 100644 --- a/src/server/services/contentful.js +++ b/src/server/services/contentful.js @@ -297,7 +297,10 @@ export async function pollArticlesForThrive() { } } // schedule a repeat each TC_EDU_POLL_TIME - setTimeout(pollArticlesForThrive, config.TC_EDU_POLL_TIME * 1000).unref(); + setTimeout( + pollArticlesForThrive, + config.TC_EDU_POLL_TIME * 1000 + (Math.floor(Math.random() * Math.floor(10)) * 60 * 1000), + ).unref(); } // /* Contentful CDN service. */ From 109db581002f93c2507e7bb778d1b5d8fdef6117 Mon Sep 17 00:00:00 2001 From: Kiril Kartunov Date: Mon, 25 Nov 2019 22:27:44 +0200 Subject: [PATCH 3/4] Poll filter and time interval --- config/default.js | 1 - src/server/services/contentful.js | 175 ++++++++++++++++-------------- 2 files changed, 92 insertions(+), 84 deletions(-) diff --git a/config/default.js b/config/default.js index 4b991f9cc4..4475c421f2 100644 --- a/config/default.js +++ b/config/default.js @@ -266,5 +266,4 @@ module.exports = { TC_EDU_ARTICLES_PATH: '/articles', TC_EDU_SEARCH_PATH: '/search', TC_EDU_SEARCH_BAR_MAX_RESULTS_EACH_GROUP: 3, - TC_EDU_POLL_TIME: 3600, // 1h in seconds }; diff --git a/src/server/services/contentful.js b/src/server/services/contentful.js index 2681451ddc..39a4c39f2a 100644 --- a/src/server/services/contentful.js +++ b/src/server/services/contentful.js @@ -202,6 +202,16 @@ export async function pollArticlesForThrive() { // make this function execute only when in production if (process.env.BABEL_ENV !== 'production') return; logger.log('polling blog articles for THRIVE -> INIT'); + // Create Contentful client to work with + const client = contentful.createClient({ + accessToken: config.SECRET.CONTENTFUL.MANAGEMENT_TOKEN, + }); + // connect to Thrive space + const env = await client.getSpace(config.SECRET.CONTENTFUL.EDU.SPACE_ID) + .then(space => space.getEnvironment('master')); + // fetch the poll interval + const pollConf = await env.getEntry('7n1HT3MhUhgjvCzd36Nymd'); + const pollInt = pollConf ? pollConf.fields.timeInSeconds : 86400; // default to day if not found // fetch the RSS feed and parse it const feedXML = await fetch(config.URL.THRIVE_POLL_FEED); if (feedXML.ok) { @@ -209,97 +219,96 @@ export async function pollArticlesForThrive() { feed = await Promise.resolve(xml2json.toJson(feed, { object: true })); if (feed) { // when feed loaded and parsed ok - // connect to Thrive space - const client = contentful.createClient({ - accessToken: config.SECRET.CONTENTFUL.MANAGEMENT_TOKEN, - }); - client.getSpace(config.SECRET.CONTENTFUL.EDU.SPACE_ID) - .then(space => space.getEnvironment('master')) - // loop all feed items and check if those exists in space - // if not existing then create - .then(env => Promise.all( - _.map(feed.rss.channel.item, - blogPost => env.getEntries({ - content_type: 'article', - 'fields.title[match]': blogPost.title, - }) - .then((queryData) => { - if (queryData.total === 0) { - // article not found in Contentful space - // will create it with payload... - const article = { - fields: { - title: { 'en-US': blogPost.title }, - type: { 'en-US': 'Article' }, - tags: { 'en-US': blogPost.category }, - creationDate: { 'en-US': new Date(blogPost.pubDate) }, - content: { 'en-US': blogPost.description }, - externalArticle: { 'en-US': true }, - contentUrl: { 'en-US': blogPost.link }, - }, - }; - // check if author exists - // if yes link it if no create it? - return env.getEntries({ - content_type: 'person', - query: blogPost['dc:creator'], - }) - .then((author) => { - // found an author that matches link it - if (author.total) { - article.fields.contentAuthor = { - 'en-US': [{ - sys: { - type: 'Link', linkType: 'Entry', id: author.items[0].sys.id, - }, - }], - }; - } - // try get the page to extract its featured image - return fetch(blogPost.link) - .then(rsp => rsp.text()) - .then((HTMLsrc) => { - const match = /]*href="([^"]+)"/gm.exec(HTMLsrc); - if (match && match[1]) { - // create the asset in Contentful - return env.createAsset({ - fields: { - title: { 'en-US': blogPost.title }, - file: { - 'en-US': { fileName: blogPost.title, contentType: 'image', upload: match[1] }, - }, + // apply filter first based on tags work only with 'Community Stories' + feed.rss.channel.item = _.filter( + feed.rss.channel.item, + blogItem => blogItem.category.indexOf('Community Stories') !== -1, + ); + // loop all feed items and check if those exists in space + // if not existing then create + Promise.all( + _.map(feed.rss.channel.item, + blogPost => env.getEntries({ + content_type: 'article', + 'fields.title[match]': blogPost.title, + }) + .then((queryData) => { + if (queryData.total === 0) { + // article not found in Contentful space + // will create it with payload... + const article = { + fields: { + title: { 'en-US': blogPost.title }, + type: { 'en-US': 'Article' }, + tags: { 'en-US': blogPost.category }, + creationDate: { 'en-US': new Date(blogPost.pubDate) }, + content: { 'en-US': blogPost.description }, + externalArticle: { 'en-US': true }, + contentUrl: { 'en-US': blogPost.link }, + }, + }; + // check if author exists + // if yes link it if no create it? + return env.getEntries({ + content_type: 'person', + query: blogPost['dc:creator'], + }) + .then((author) => { + // found an author that matches link it + if (author.total) { + article.fields.contentAuthor = { + 'en-US': [{ + sys: { + type: 'Link', linkType: 'Entry', id: author.items[0].sys.id, + }, + }], + }; + } + // try get the page to extract its featured image + return fetch(blogPost.link) + .then(rsp => rsp.text()) + .then((HTMLsrc) => { + const match = /]*href="([^"]+)"/gm.exec(HTMLsrc); + if (match && match[1]) { + // create the asset in Contentful + return env.createAsset({ + fields: { + title: { 'en-US': blogPost.title }, + file: { + 'en-US': { fileName: blogPost.title, contentType: 'image', upload: match[1] }, }, - }) - .then(asset => asset.processForAllLocales()) - .then((asset) => { - article.fields.featuredImage = { - 'en-US': { - sys: { - type: 'Link', linkType: 'Asset', id: asset.sys.id, - }, + }, + }) + .then(asset => asset.processForAllLocales()) + .then((asset) => { + article.fields.featuredImage = { + 'en-US': { + sys: { + type: 'Link', linkType: 'Asset', id: asset.sys.id, }, - }; - return env.createEntry('article', article); - }); - } - // could not find image - // just create the article without it... - return env.createEntry('article', article); - }); - }); - } - // Article exists in Contentful space - // do nothing... - return 1; - })), - )) + }, + }; + return env.createEntry('article', article); + }); + } + // could not find image + // just create the article without it... + return env.createEntry('article', article); + }); + }); + } + // Article exists in Contentful space + // do nothing... + return 1; + })), + ) .then(() => logger.log('polling blog articles for THRIVE -> DONE')); } } // schedule a repeat each TC_EDU_POLL_TIME setTimeout( pollArticlesForThrive, - config.TC_EDU_POLL_TIME * 1000 + (Math.floor(Math.random() * Math.floor(10)) * 60 * 1000), + pollInt * 1000 + (Math.floor(Math.random() * Math.floor(5)) * 60 * 1000), ).unref(); } From f945794d191997781ac0619d26bbe5e840cb321c Mon Sep 17 00:00:00 2001 From: Sushil Shinde Date: Tue, 26 Nov 2019 12:30:39 +0530 Subject: [PATCH 4/4] deploying on test --- .circleci/config.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 58e22a70aa..4a2548b114 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -176,7 +176,6 @@ workflows: only: - develop - hot-fix - - feature-contentful - survey_tc - fix-groups-updates - sort_registrations_submissions @@ -185,8 +184,9 @@ workflows: context : org-global filters: branches: - only: - - develop + only: + - develop + - feature-contentful # This is beta env for production soft releases - "build-prod-beta": context : org-global @@ -194,12 +194,6 @@ workflows: branches: only: - develop - - feature-contentful - - survey_tc - - feature-groups-api - - mm-broker-api - - fix-groups-updates - - sort_registrations_submissions # Production builds are exectuted only on tagged commits to the # master branch. - "build-prod":