You are on page 1of 2

const fs = require('fs').

promises;

const { GENERATED_FILES_PATHS } = require('../../constants/paths');

const { getModel, getContentModelIds } = require('$models/helpers');

class RobotsTxtService {
constructor(site) {
if (!site) {
throw new Error('RobotsService: Missing site');
}

this.site = site;
this.siteId = site._id;
this.targetPath = `${GENERATED_FILES_PATHS.ROBOTS_TXT}/${site._id}.txt`;

this.robotsContent = ``;
}

async generate() {
await this.#handleCommonRules();
this.#handleSocialNetworks();
this.#handleSitemaps();
await this.#writeRobotsFile();
}

async #writeRobotsFile() {
try {
await fs.writeFile(this.targetPath, this.robotsContent);

console.log(
`robots.txt file written for ${this.siteId} at path : ${this.targetPath}`
);

return true;
} catch (err) {
console.error(err);
}
}

async #handleCommonRules() {
this.robotsContent = `
User-agent: *
Disallow: /_next/image
Disallow: /api
Disallow: /search
`;
await this.#handlePagesToNotIndex();
}

async #handlePagesToNotIndex() {
// Pages
const Page = getModel('Page')(this.siteId);
const pages = await Page.find({ disableSeoIndexation: true });

const contentModels = getContentModelIds();


let contents = [];
for (const modelName of contentModels) {
const Model = getModel(modelName)(this.siteId);
contents = [
...contents,
...(await Model.find({ disableSeoIndexation: true }).isPublished()),
];
}

const forbiddenUrls = await Promise.all(


[...pages, ...contents].map((content) => content.getFrontendFullPath())
);
for (const url of forbiddenUrls) {
if (!url) continue;
this.robotsContent += `Disallow: ${url}\n`;
}
}

#handleSocialNetworks() {
this.robotsContent += `\n
User-agent: Twitterbot
Allow: /_next/image
Allow: /api/image
Allow: /api/upload

User-agent: facebookexternalhit/1.1
Allow: /_next/image
Allow: /api/image
Allow: /api/upload
`;
}

#handleSitemaps() {
this.robotsContent += `\nSitemap: ${this.site.getFrontUrl()}/sitemap.xml\n`;
if (this.site.isIndexedByGoogleNews) {
this.robotsContent += `Sitemap: ${this.site.getFrontUrl()}/sitemap-google-
news.xml\n`;
}
}
}

module.exports = RobotsTxtService;

You might also like