Gatsby build ignores few images to copy into static folder - gatsby-image

Question

I have uploads folder with all my assets. When I run gatsby clean / gatsby build, it copies most of the images except 4 of them. It doesn't throw any error or warning as to what it ignores them. The same command on another docker container copies all of the images. The node/gatsby/npm and package.json are precisely the same on both containers. Am I missing something?
Package.json:
{
"name": "gatsby-starter-hello-world",
"private": true,
"description": "A simplified bare-bones starter for Gatsby",
"version": "0.1.0",
"license": "MIT",
"scripts": {
"build": "gatsby build",
"clean": "gatsby clean",
"develop": "gatsby develop -H 0.0.0.0",
"format": "prettier --write \"**/*.{js,jsx,json,md}\"",
"start": "npm run develop",
"refresh": "curl -X POST http://localhost:8000/__refresh",
"docker": "docker-compose -f docker-dev.yml up",
"serve": "gatsby serve -H 0.0.0.0 --port 8000",
"db:ql": "node graphql-server.js",
"test": "echo \"Write tests! -> https://gatsby.dev/unit-testing\" && exit 1"
},
"dependencies": {
"#apollo/react-hooks": "^3.1.2",
"#babel/runtime": "^7.8.4",
"#hot-loader/react-dom": "^16.11.0",
"#use-it/interval": "^0.1.3",
"apollo-boost": "^0.4.7",
"axios": "^0.19.0",
"body-scroll-lock": "^2.6.4",
"bootstrap": "^4.4.1",
"clsx": "^1.0.4",
"cross-env": "^6.0.3",
"final-form": "^4.18.5",
"final-form-calculate": "^1.3.1",
"final-form-set-field-data": "^1.0.2",
"gatsby": "^2.19.10",
"gatsby-image": "^2.2.39",
"gatsby-plugin-alias-imports": "^1.0.5",
"gatsby-plugin-client-side-redirect": "0.0.2",
"gatsby-plugin-lodash": "^3.1.19",
"gatsby-plugin-manifest": "^2.2.40",
"gatsby-plugin-minify-classnames": "^0.1.2",
"gatsby-plugin-offline": "^3.0.33",
"gatsby-plugin-polyfill-io": "^1.1.0",
"gatsby-plugin-react-axe": "^0.3.0",
"gatsby-plugin-react-helmet": "^3.1.21",
"gatsby-plugin-robots-txt": "^1.5.0",
"gatsby-plugin-s3": "^0.3.2",
"gatsby-plugin-sass": "^2.1.27",
"gatsby-plugin-sharp": "^2.4.4",
"gatsby-plugin-sitemap": "^2.2.27",
"gatsby-plugin-webpack-bundle-analyzer": "^1.0.5",
"gatsby-source-filesystem": "^2.1.47",
"gatsby-source-graphql": "^2.5.1",
"gatsby-source-wordpress": "^3.3.1",
"gatsby-transformer-sharp": "^2.3.13",
"gatsby-wordpress-inline-images": "^1.2.1",
"gatsby-wpgraphql-inline-images": "^0.2.5",
"graphql": "^14.6.0",
"graphql-tag": "^2.10.1",
"html-react-parser": "^0.10.0",
"install": "^0.13.0",
"isomorphic-fetch": "^2.2.1",
"lodash": "^4.17.15",
"lodash-es": "^4.17.15",
"moment": "^2.24.0",
"node-sass": "^4.13.0",
"prop-types": "^15.7.2",
"query-string": "^6.8.3",
"react": "^16.12.0",
"react-apollo": "^3.1.2",
"react-autosuggest": "^9.4.3",
"react-bootstrap": "^1.0.0-beta.16",
"react-calendar": "^2.19.2",
"react-datepicker": "^2.11.0",
"react-dates": "^21.8.0",
"react-dom": "^16.12.0",
"react-final-form": "^6.3.5",
"react-helmet": "^5.2.1",
"react-hotjar": "^2.2.0",
"react-icons": "^3.8.0",
"react-image-lightbox": "^5.1.1",
"react-payment-inputs": "^1.0.7",
"react-select": "^3.0.8",
"react-star-ratings": "^2.3.0",
"react-sticky": "^6.0.3",
"styled-components": "^4.4.1"
},
"devDependencies": {
"express": "^4.17.1",
"json-graphql-server": "^2.1.3",
"prettier": "^1.18.2"
},
"repository": {
"type": "git",
"url": "https://github.com/gatsbyjs/gatsby-starter-hello-world"
},
"bugs": {
"url": "https://github.com/gatsbyjs/gatsby/issues"
},
"browserslist": [
"> 2%",
"last 2 versions",
"IE 11"
],
"prettier": {
"tabWidth": 4,
"semi": false
},
"deploy": "gatsby-plugin-s3 deploy --yes"
}
gatsby-config:
/**
* Configure your Gatsby site with this file.
*
* See: https://www.gatsbyjs.org/docs/gatsby-config/
*/
const fs = require('fs')
const path = require("path")
const _ = require("lodash")
const axios = require("axios")
require("dotenv").config({
path: `.env.${process.env.NODE_ENV}`,
})
const rootPath = path.resolve(__dirname)
// https://gist.github.com/flipace/bed6b89aed5cb0e19cde
function replaceLocalhostBaseUrl(object) {
const newObject = _.clone(object);
_.each(object, (val, key) => {
if (typeof val === 'string' && /(http)(.*)(localhost)(:[0-9]{4})?/.test(val)) {
newObject[key] = val.replace(/(http)(.*)(localhost)(:[0-9]{4})?/, `https://${process.env.WEBSITE_DOMAIN}`)
} else if (typeof val === 'object') {
newObject[key] = replaceLocalhostBaseUrl(val)
} else {
newObject[key] = val
}
});
return newObject;
}
function replaceAbsoluteUrls(contentString){
const regex = /href\s*=\s*(['"])(https?:\/\/.+?)\1/ig
let link
let l
const getLocation = function(href){
let l = document.createElement("a")
l.href = href
return l
}
while((link = regex.exec(contentString)) !== null) {
//only replace urls where localhost or awesomenz.com is mentioned, like uat.awesomenz, dev.awesomenz
if (/localhost|dummy.co.nz/.test(link[2]))
{
l = getLocation(link[2])
contentString = contentString.replace(link[2],
// process.env.SITE_URL +
l.pathname +
"?" +l.href.substring( l.href.indexOf("?")+1 )
);
}
}
return contentString
}
function replaceImagesToLocalEnv(tg){
let fromUrl = "https://uat.dummy.co.nz"
if(process.env.ENV_TYPE == "build")
{
tg.url = tg.url.replace(fromUrl, process.env.MEDIA_BASE_SOURCE_URL)
tg.link = tg.link.replace(fromUrl, process.env.MEDIA_BASE_SOURCE_URL)
tg.icon = tg.icon.replace(fromUrl, process.env.MEDIA_BASE_SOURCE_URL)
tg.sizes.thumbnail = tg.sizes.thumbnail.replace(fromUrl, process.env.MEDIA_BASE_SOURCE_URL)
tg.sizes.medium = tg.sizes.medium.replace(fromUrl, process.env.MEDIA_BASE_SOURCE_URL)
tg.sizes.medium_large = tg.sizes.medium_large.replace(fromUrl, process.env.MEDIA_BASE_SOURCE_URL)
tg.sizes.large = tg.sizes.large.replace(fromUrl, process.env.MEDIA_BASE_SOURCE_URL)
tg.sizes['1536x1536'] = tg.sizes['1536x1536'].replace(fromUrl, process.env.MEDIA_BASE_SOURCE_URL)
tg.sizes['2048x2048'] = tg.sizes['2048x2048'].replace(fromUrl, process.env.MEDIA_BASE_SOURCE_URL)
}
return tg
}
const download_image = async (url, image_path) => {
const imagePathDir = path.dirname(image_path)
await fs.promises.mkdir(imagePathDir, { recursive: true }).catch(console.error);
return axios({
url,
responseType: 'stream',
}).then(
response =>
new Promise((resolve, reject) => {
response.data
.pipe(fs.createWriteStream(image_path))
.on('finish', () => resolve())
.on('error', e => reject(e));
}),
);
}
function sleep(ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
let wasOgDefaultImageDownloaded = false
let wasOgFrontpageImageDownloaded = false
async function downloadSocialImages(meta) {
if(meta.yoast_wpseo_facebook_image) {
const image = meta.yoast_wpseo_facebook_image
const imageUrl = new URL(image)
await download_image(image, path.join(rootPath, 'public', imageUrl.pathname))
}
if(meta.yoast_wpseo_twitter_image) {
const image = meta.yoast_wpseo_twitter_image
const imageUrl = new URL(image)
await download_image(image, path.join(rootPath, 'public', imageUrl.pathname))
}
if(!wasOgDefaultImageDownloaded && meta.yoast_wpseo_social_defaults.og_default_image) {
wasOgDefaultImageDownloaded = true
const image = meta.yoast_wpseo_social_defaults.og_default_image
const imageUrl = new URL(image)
await download_image(image, path.join(rootPath, 'public', imageUrl.pathname))
}
if(!wasOgFrontpageImageDownloaded && meta.yoast_wpseo_social_defaults.og_frontpage_image) {
wasOgFrontpageImageDownloaded = true
const image = meta.yoast_wpseo_social_defaults.og_frontpage_image
const imageUrl = new URL(image)
await download_image(image, path.join(rootPath, 'public', imageUrl.pathname))
}
}
module.exports = {
siteMetadata: {
title: `dummy`,
// siteUrl is hard coded for use in sitemap.xml (`gatsby-plugin-sitemap`)
siteUrl: process.env.SITE_URL,
},
plugins: [
{
resolve: 'gatsby-plugin-robots-txt',
options: {
host: process.env.SITE_URL,
sitemap: `${process.env.SITE_URL}/sitemap.xml`,
policy: [
{
userAgent: '*',
allow: '/',
disallow: [
'/cart',
'/checkout',
'/filter',
'/filter-tours',
'/manage-booking',
'/mydetails',
'/dev',
'/admin',
'/?flush',
]
},
]
}
},
{
resolve: `gatsby-plugin-polyfill-io`,
options: {
features: [`Array.prototype.find`, 'String.prototype.padStart'],
},
},
`gatsby-plugin-lodash`,
{
resolve: `gatsby-plugin-alias-imports`,
options: {
alias: {
// 'react-dom': '#hot-loader/react-dom',
"entrada-ui": path.join(rootPath, "src/entrada-ui"),
components: path.join(rootPath, "src/components"),
constants: path.join(rootPath, "src/constants"),
queries: path.join(rootPath, "src/queries"),
utils: path.join(rootPath, "src/utils"),
images: path.join(rootPath, "src/images"),
css: path.join(rootPath, "src/css"),
theme: path.join(rootPath, "src/theme"),
},
extensions: [],
},
},
{
resolve: "gatsby-plugin-sass",
options: {
cssLoaderOptions: {
camelCase: true,
},
},
},
{
resolve: `gatsby-plugin-minify-classnames`,
options: {
develop: false, // Enable/Disable on `gatsby develop`
},
},
// {
// resolve: "gatsby-source-graphql",
// options: {
// typeName: "PIH",
// fieldName: "pih",
// url: "https://api.dev.tourcatalogue.co.nz/supplier-graphql-api/query",
// // // HTTP headers
// // headers: {
// // // Learn about environment variables: https://gatsby.dev/env-vars
// // Authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
// // },
// // // Additional options to pass to node-fetch
// // fetchOptions: {},
// },
// },
// {
// resolve: "gatsby-source-graphql",
// options: {
// typeName: "WP",
// fieldName: "wp",
// url: "http://localhost:4100/graphql",
// },
// },
// Wordpress
`gatsby-plugin-react-helmet`,
{
resolve: `gatsby-source-filesystem`,
options: {
name: `images`,
path: `${__dirname}/src/images`,
},
},
`gatsby-transformer-sharp`,
`gatsby-plugin-sharp`,
// {
// resolve: `gatsby-plugin-manifest`,
// options: {
// name: `gatsby-starter-default`,
// short_name: `starter`,
// start_url: `/`,
// background_color: `#663399`,
// theme_color: `#663399`,
// display: `minimal-ui`,
// icon: `src/images/gatsby-icon.png`, // This path is relative to the root of the site.
// crossOrigin: `use-credentials`,
// },
// },
{
resolve: "gatsby-source-wordpress",
options: {
baseUrl: `${process.env.WORDPRESS_URL}`,
protocol: `${process.env.WP_PROTOCOL}`,
plugins: [
{
resolve: `gatsby-wordpress-inline-images`,
options: {
baseUrl: `${process.env.WORDPRESS_URL}`,
protocol: `${process.env.WP_PROTOCOL}`
// baseUrl: `uat.dummy.co.nz`,
// protocol: `https`
}
}
],
// TODO: Need to have further testing
// searchAndReplaceContentUrls: {
// sourceUrl: `${process.env.SITE_URL}`,
// replacementUrl: "",
// },
hostingWPCOM: false,
useACF: true,
acfOptionPageIds: [],
verboseOutput: false,
perPage: 100,
// Set how many simultaneous requests are sent at once.
concurrentRequests: 10,
includedRoutes: [
"**/categories",
"**/posts",
"**/pages",
"**/media",
"**/tags",
"**/tours",
"**/destinations",
"**/themes",
"**/taxonomies",
"**/announcements",
"**/*/*/menus", // <== Menu api endpoint
"**/*/*/menu-locations", // <== Menu api endpoint
"**/exposeredirections", // <== Exposeredirections api endpoint
],
excludedRoutes: [],
normalizers: normalizers => {
return _.compact([
process.env.MEDIA_BASE_SOURCE_URL ?
{
name: `changeMediaSourceUrls`,
normalizer: async ({
entities,
}) => (
entities.map((entity) => {
if(entity.content && entity.content !== ""){
entity.content = replaceAbsoluteUrls(entity.content)
}
///////////////////////////////////////////////////////////
if(entity.acf)
{
if(entity.acf.gallery)
{
let nLoop = entity.acf.gallery.length
if( nLoop > 0)
{
let tempGallery = []
for(let c=0; c<nLoop; c++){
tempGallery.push( replaceImagesToLocalEnv(entity.acf.gallery[c]) )
}
entity.acf.gallery = tempGallery
}
}
if(entity.acf.card_image)
{
entity.acf.card_image = replaceImagesToLocalEnv(entity.acf.card_image)
}
if(entity.acf.card_map)
{
entity.acf.card_map = replaceImagesToLocalEnv(entity.acf.card_map)
}
if(entity.acf.image)
{
entity.acf.image = replaceImagesToLocalEnv(entity.acf.image)
}
if(entity.acf.info_cards)
{
let nLoop = entity.acf.info_cards.length
if( nLoop > 0)
{
let tempCardsInfo = []
let tmp
for(let c=0; c<nLoop; c++){
tmp = entity.acf.info_cards[c]
tmp.card_image = replaceImagesToLocalEnv(tmp.card_image)
tempCardsInfo.push( tmp )
}
entity.acf.info_cards = tempCardsInfo
}
}
}
///////////////////////////////////////////////////////////
if(entity.__type === 'wordpress__wp_media') {
// Transforms:
// https://uat.dummy.co.nz/wp-content/uploads/2020/01/Placeholder-Image.png
// into:
// http://localhost/wp-content/uploads/2020/01/Placeholder-Image.png
entity.source_url = `${process.env.MEDIA_BASE_SOURCE_URL}/${entity.source_url.split("/").slice(3).join('/')}`
} else if (entity.yoast_meta) {
downloadSocialImages(entity.yoast_meta)
entity.yoast_meta.yoast_schema = typeof entity.yoast_meta.yoast_schema === 'string' ? JSON.parse(entity.yoast_meta.yoast_schema) : entity.yoast_meta.yoast_schema
entity.yoast_meta = replaceLocalhostBaseUrl(entity.yoast_meta)
entity.yoast_meta.yoast_schema = JSON.stringify(entity.yoast_meta.yoast_schema)
}
return entity
})
)
} : undefined,
...normalizers
])
},
},
},
// {
// resolve: 'gatsby-wpgraphql-inline-images',
// options: {
// wordPressUrl: process.env.MEDIA_BASE_SOURCE_URL+'/',
// uploadsUrl: process.env.MEDIA_BASE_SOURCE_URL+'/wp-content/uploads/',
// processPostTypes: ['Page', 'Post', 'CustomPost'],
// graphqlTypeName: 'WPGraphQL',
// httpHeaders: {
// Authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
// }
// },
// },
{
resolve: `gatsby-plugin-s3`,
options: {
bucketName: 'static-dummy'
},
},
{
resolve: `gatsby-plugin-sitemap`,
options: {
// See: https://github.com/isaacs/minimatch
// The example below will exclude the single `path/to/page` and all routes beginning with `category`
query: `
{
site {
siteMetadata {
siteUrl
}
}
allSitePage {
edges {
node {
path
}
}
}
}`,
serialize: ({ site, allSitePage }) =>
allSitePage.edges.map(edge => {
return {
url: site.siteMetadata.siteUrl + edge.node.path,
changefreq: `daily`,
priority: 0.7,
}
})
}
},
`gatsby-plugin-client-side-redirect`, // keep it in last in list,
// {
// resolve: 'gatsby-plugin-react-axe',
// options: {
// // Integrate react-axe in production. This defaults to false.
// showInProduction: false,
// },
// },
// {
// resolve: 'gatsby-plugin-webpack-bundle-analyzer',
// options: {
// analyzerPort: 8888,
// production: true,
// },
// }
],
}
env file:
# If you are running `npm run build` in your local, you need to add the port :8080
# to WORDPRESS_URL and MEDIA_BASE_SOURCE_URL
WORDPRESS_URL=localhost
MEDIA_BASE_SOURCE_URL=http://localhost
WP_POD_PUBLIC_URL=https://uat.dummy.co.nz
WP_PROTOCOL=http
ENV_TYPE=build
WEBSITE_DOMAIN=uat.dummy.co.nz
SITE_URL=https://www.uat.dummy.co.nz
GATSBY_CPU_COUNT=logical_cores