fix(gatsby-source-wordpress): adjust how endpoint urls are constructed to fix fetching for wordpress.com hosted sites and proxied urls (#10624)

## Description

As reported in #10427 some WordPress.com sites return a `_links.$.self` with a fullpath `https://public-api.wordpress.com/wp/v2/sites/$site/users/me` and others without `https://public-api.wordpress.com/`. This means it is not a reliable way of removing the `baseUrl` from the `fullUrl`.

Instead we pass the route key in as the `fullPath` rather than the `fullUrl`. We then extract the `basePath` from the `baseUrl` before finally removing the `basePath` from the `fullPath`.

We also need to extract the raw entity type from the `fullPath` rather than route.

We also need to build the `fullUrl` from `baseUrl` and `fullPath` so I introduced the `buildFullUrl` function to do just that.

I’ve included tests for both WordPress.com and WordPress.org.

## Related Issues

Fixes #10427.

Author: lukemorton

packages/gatsby-source-wordpress/src/__tests__/fetch.js

@@ -0,0 +1,41 @@
+const fetch = require(`../fetch`)
+
+describe(`Fetching`, () => {
+  it(`gets raw entity type for wordpress.com`, () => {
+    const key = `/wp/v2/sites/example.wordpress.com/posts`
+    const path = fetch.getRawEntityType(key)
+    expect(path).toEqual(`posts`)
+  })
+
+  it(`gets route path for wordpress.com by removing base path from the full path`, () => {
+    const key = `/wp/v2/sites/example.wordpress.com/posts`
+    const baseUrl = `https://public-api.wordpress.com/wp/v2/sites/example.wordpress.com`
+    const path = fetch.getRoutePath(baseUrl, key)
+    expect(path).toEqual(`/posts`)
+  })
+
+  it(`gets route path for wordpress.org by removing base path from the full path`, () => {
+    const key = `/wp-json/wp/v2/posts`
+    const baseUrl = `http://dev-gatbsyjswp.pantheonsite.io/wp-json`
+    const path = fetch.getRoutePath(baseUrl, key)
+    expect(path).toEqual(`/wp/v2/posts`)
+  })
+
+  it(`builds full URL correctly for wordpress.com`, () => {
+    const key = `/wp/v2/sites/example.wordpress.com/posts`
+    const baseUrl = `https://public-api.wordpress.com/wp/v2/sites/example.wordpress.com`
+    const fullUrl = fetch.buildFullUrl(baseUrl, key, true)
+    expect(fullUrl).toEqual(
+      `https://public-api.wordpress.com/wp/v2/sites/example.wordpress.com/posts`
+    )
+  })
+
+  it(`builds full URL correctly for wordpress.org`, () => {
+    const key = `/wp/v2/posts`
+    const baseUrl = `http://dev-gatbsyjswp.pantheonsite.io/wp-json`
+    const fullUrl = fetch.buildFullUrl(baseUrl, key, false)
+    expect(fullUrl).toEqual(
+      `http://dev-gatbsyjswp.pantheonsite.io/wp-json/wp/v2/posts`
+    )
+  })
+})

packages/gatsby-source-wordpress/src/fetch.js

@@ -2,6 +2,7 @@ const querystring = require(`querystring`)
 const axios = require(`axios`)
 const _ = require(`lodash`)
 const minimatch = require(`minimatch`)
+const { URL } = require(`url`)
 const colorized = require(`./output-color`)
 const httpExceptionHandler = require(`./http-exception-handler`)
 const requestInQueue = require(`./request-in-queue`)
@@ -133,6 +134,7 @@ Mama Route URL: ${url}
       url,
       _verbose,
       _useACF,
+      _hostingWPCOM,
       _acfOptionPageIds,
       _includedRoutes,
       _excludedRoutes,
@@ -503,7 +505,7 @@ function getValidRoutes({
 
     // A valid route exposes its _links (for now)
     if (route._links) {
-      const entityType = getRawEntityType(route)
+      const entityType = getRawEntityType(key)
 
       // Excluding the "technical" API Routes
       const excludedTypes = [
@@ -517,7 +519,8 @@ function getValidRoutes({
         `/jwt-auth/**`,
       ]
 
-      const routePath = getRoutePath(url, route._links.self)
+      const routePath = getRoutePath(url, key)
+
       const whiteList = _includedRoutes
       const blackList = [...excludedTypes, ..._excludedRoutes]
 
@@ -564,7 +567,11 @@ function getValidRoutes({
             )}_${entityType.replace(/-/g, `_`)}`
             break
         }
-        validRoutes.push({ url: route._links.self, type: validType })
+
+        validRoutes.push({
+          url: buildFullUrl(url, key, _hostingWPCOM),
+          type: validType,
+        })
       } else {
         if (_verbose) {
           const invalidType = inBlackList ? `blacklisted` : `not whitelisted`
@@ -591,23 +598,40 @@ function getValidRoutes({
 }
 
 /**
- * Extract the raw entity type from route
+ * Extract the raw entity type from fullPath
  *
- * @param {any} route
+ * @param {any} full path to extract raw entity from
  */
-const getRawEntityType = route =>
-  route._links.self.substring(
-    route._links.self.lastIndexOf(`/`) + 1,
-    route._links.self.length
-  )
+const getRawEntityType = fullPath =>
+  fullPath.substring(fullPath.lastIndexOf(`/`) + 1, fullPath.length)
 
 /**
  * Extract the route path for an endpoint
  *
  * @param {any} baseUrl The base site URL that should be removed
- * @param {any} fullUrl The full URL to retrieve the route path from
+ * @param {any} fullPath The full path to retrieve the route path from
  */
-const getRoutePath = (baseUrl, fullUrl) => fullUrl.replace(baseUrl, ``)
+const getRoutePath = (baseUrl, fullPath) => {
+  const baseUrlObj = new URL(baseUrl)
+  const basePath = baseUrlObj.pathname
+  return fullPath.replace(basePath, ``)
+}
+
+/**
+ * Build full URL from baseUrl and fullPath.
+ * Method of contructing full URL depends on wether it's hosted on wordpress.com
+ * or not as wordpress.com have slightly different (custom) REST structure
+ *
+ * @param {any} baseUrl The base site URL that should be prepended to full path
+ * @param {any} fullPath The full path to build URL from
+ * @param {boolean} _hostingWPCOM Is hosted on wordpress.com
+ */
+const buildFullUrl = (baseUrl, fullPath, _hostingWPCOM) => {
+  if (_hostingWPCOM) {
+    baseUrl = new URL(baseUrl).origin
+  }
+  return `${baseUrl}${fullPath}`
+}
 
 /**
  * Extract the route manufacturer
@@ -617,4 +641,7 @@ const getRoutePath = (baseUrl, fullUrl) => fullUrl.replace(baseUrl, ``)
 const getManufacturer = route =>
   route.namespace.substring(0, route.namespace.lastIndexOf(`/`))
 
+fetch.getRawEntityType = getRawEntityType
+fetch.getRoutePath = getRoutePath
+fetch.buildFullUrl = buildFullUrl
 module.exports = fetch