Professional Documents
Culture Documents
promises;
class RobotsTxtService {
constructor(site) {
if (!site) {
throw new Error('RobotsService: Missing site');
}
this.site = site;
this.siteId = site._id;
this.targetPath = `${GENERATED_FILES_PATHS.ROBOTS_TXT}/${site._id}.txt`;
this.robotsContent = ``;
}
async generate() {
await this.#handleCommonRules();
this.#handleSocialNetworks();
this.#handleSitemaps();
await this.#writeRobotsFile();
}
async #writeRobotsFile() {
try {
await fs.writeFile(this.targetPath, this.robotsContent);
console.log(
`robots.txt file written for ${this.siteId} at path : ${this.targetPath}`
);
return true;
} catch (err) {
console.error(err);
}
}
async #handleCommonRules() {
this.robotsContent = `
User-agent: *
Disallow: /_next/image
Disallow: /api
Disallow: /search
`;
await this.#handlePagesToNotIndex();
}
async #handlePagesToNotIndex() {
// Pages
const Page = getModel('Page')(this.siteId);
const pages = await Page.find({ disableSeoIndexation: true });
#handleSocialNetworks() {
this.robotsContent += `\n
User-agent: Twitterbot
Allow: /_next/image
Allow: /api/image
Allow: /api/upload
User-agent: facebookexternalhit/1.1
Allow: /_next/image
Allow: /api/image
Allow: /api/upload
`;
}
#handleSitemaps() {
this.robotsContent += `\nSitemap: ${this.site.getFrontUrl()}/sitemap.xml\n`;
if (this.site.isIndexedByGoogleNews) {
this.robotsContent += `Sitemap: ${this.site.getFrontUrl()}/sitemap-google-
news.xml\n`;
}
}
}
module.exports = RobotsTxtService;