Creating a website aggregator with ChatGPT, React, and Node.js

mkdir website-aggregatorcd website-aggregatormkdir client server

cd server & npm init -y

npm install express cors nodemon

touch index.js

//index.jsconst express = require("express");const cors = require("cors");const app = express();const PORT = 4000;app.use(express.urlencoded({ extended: true }));app.use(express.json());app.use(cors());app.get("/api", (req, res) => {    res.json({        message: "Hello world",    });});app.listen(PORT, () => {    console.log(`Server listening on ${PORT}`);});

npm install chatgpt puppeteer

{ "type": "module" }

import { createRequire } from "module";const require = createRequire(import.meta.url);//...other code statements

//In server/package.json"scripts": {    "test": "echo \"Error: no test specified\" && exit 1",    "start": "nodemon index.js"  },

npm start

cd clientnpx create-react-app ./

function App() {    return (        <div>            <p>Hello World!</p>        </div>    );}export default App;

@import url("https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@300;400;500;600;700&display=swap");* {    box-sizing: border-box;    margin: 0;    padding: 0;    font-family: "Space Grotesk", sans-serif;}input {    padding: 10px;    width: 70%;    margin: 10px 0;    outline: none;}button {    width: 200px;    cursor: pointer;    padding: 10px 20px;    outline: none;    border: none;    background-color: #6d9886;}.home,.home__form,.website__item,.loading {    display: flex;    align-items: center;    justify-content: center;    flex-direction: column;}.home__form > h2,.home__form {    margin-bottom: 30px;    text-align: center;    width: 100%;}.home {    min-height: 100vh;    padding: 20px;    width: 100%;}.website__container {    width: 100%;    min-height: 50vh;    border-radius: 5px;    display: flex;    align-items: center;    justify-content: center;    flex-wrap: wrap;    padding: 15px;}.website__item {    width: 80%;    margin: 10px;    background-color: #f7f7f7;    border-radius: 5px;    padding: 30px;    box-shadow: 0 2px 8px 0 rgba(99, 99, 99, 0.2);}.website__item > img {    width: 70%;    border-radius: 5px;}.website__item > h3 {    margin: 10px 0;}.website__item > p {    text-align: center;    opacity: 0.7;}.loading {    height: 100vh;    background-color: #f2e7d5;}

import React, { useState } from "react";const App = () => {    const [url, setURL] = useState("");    const handleSubmit = (e) => {        e.preventDefault();        console.log({ url });        setURL("");    };    return (        <div className='home'>            <form className='home__form'>                <h2>Website Aggregator</h2>                <label htmlFor='url'>Provide the website URL</label>                <input                    type='url'                    name='url'                    id='url'                    value={url}                    onChange={(e) => setURL(e.target.value)}                />                <button onClick={handleSubmit}>ADD WEBSITE</button>            </form>        </div>    );};export default App;

app.post("/api/url", (req, res) => {    const { url } = req.body;    // The URL from the React app    console.log(url);});

// Import Puppeteer at the topconst puppeteer = require("puppeteer");app.post("/api/url", (req, res) => {    const { url } = req.body;    // Puppeteer webscraping function    (async () => {        const browser = await puppeteer.launch();        const page = await browser.newPage();        await page.goto(url);        // returns all the website content        const websiteContent = await page.evaluate(() => {            return document.documentElement.innerText.trim();        });        // returns the website meta image        const websiteOgImage = await page.evaluate(() => {            const metas = document.getElementsByTagName("meta");            for (let i = 0; i < metas.length; i++) {                if (metas[i].getAttribute("property") === "og:image") {                    return metas[i].getAttribute("content");                }            }        });        console.log({ websiteContent, websiteOgImage })        await browser.close();    })();});

const handleSubmit = (e) => {    e.preventDefault();    setLoading(true);    setURL("");    // Calls the function.    sendURL();};async function sendURL() {    try {        const request = await fetch("http://localhost:4000/api/url", {            method: "POST",            body: JSON.stringify({                url,            }),            headers: {                Accept: "application/json",                "Content-Type": "application/json",            },        });        const data = await request.json();        // toggles the loading state if the request is successful        if (data.message) {            setLoading(false);        }    } catch (err) {        console.error(err);    }}

const [loading, setLoading] = useState(false);

import React from "react";const Loading = () => {    return (        <div className='loading'>            <h1>Loading, please wait...</h1>        </div>    );};export default Loading;

import Loading from "./Loading";// Add this code within the App.js componentif (loading) {    return <Loading />;}

import { ChatGPTAPIBrowser } from "chatgpt";async function chatgptFunction(content) {    // use puppeteer to bypass cloudflare (headful because of captchas)    const api = new ChatGPTAPIBrowser({        email: "<CHATGPT_EMAIL_ADDRESS>",        password: "<CHATGPT_PASSWORD>",    });    await api.initSession();    // Extracts the brand name from the website content    const getBrandName = await api.sendMessage(        `I have a raw text of a website, what is the brand name in a single word? ${content}`    );    // Extracts the brand description from the website content    const getBrandDescription = await api.sendMessage(        `I have a raw text of a website, can you extract the description of the website from the raw text. I need only the description and nothing else. ${content}`    );    // Returns the response from ChatGPT    return {        brandName: getBrandName.response,        brandDescription: getBrandDescription.response,    };}

// holds all the ChatGPT resultconst database = [];// generates a random string as IDconst generateID = () => Math.random().toString(36).substring(2, 10);app.post("/api/url", (req, res) => {    const { url } = req.body;    (async () => {        const browser = await puppeteer.launch();        const page = await browser.newPage();        await page.goto(url);        const websiteContent = await page.evaluate(() => {            return document.documentElement.innerText.trim();        });        const websiteOgImage = await page.evaluate(() => {            const metas = document.getElementsByTagName("meta");            for (let i = 0; i < metas.length; i++) {                if (metas[i].getAttribute("property") === "og:image") {                    return metas[i].getAttribute("content");                }            }        });        // accepts the website content as a parameter        let result = await chatgptFunction(websiteContent);        // adds the brand image and ID to the result        result.brandImage = websiteOgImage;        result.id = generateID();    // adds the result to the array        database.push(result);    // returns the results        return res.json({            message: "Request successful!",            database,        });        await browser.close();    })();});

const [websiteContent, setWebsiteContent] = useState([]);async function sendURL() {    try {        const request = await fetch("http://localhost:4000/api/url", {            method: "POST",            body: JSON.stringify({                url,            }),            headers: {                Accept: "application/json",                "Content-Type": "application/json",            },        });        const data = await request.json();        if (data.message) {            setLoading(false);            // update the state with the server response            setWebsiteContent(data.database);        }    } catch (err) {        console.error(err);    }}

const App = () => {    //...other code statements// remove the quotation marks around the descriptionconst trimDescription = (content) =>        content.match(/(?:"[^"]*"|^[^"]*$)/)[0].replace(/"/g, "");    return (        <div className='home'>            <form className='home__form'>                <h2>Website Aggregator</h2>                <label htmlFor='url'>Provide the website URL</label>                <input                    type='url'                    name='url'                    id='url'                    value={url}                    onChange={(e) => setURL(e.target.value)}                />                <button onClick={handleSubmit}>ADD WEBSITE</button>            </form>            <main className='website__container '>                {websiteContent.map((item) => (                    <div className='website__item' key={item.id}>                        <img src={item?.brandImage} alt={item?.brandName} />                        <h3>{item?.brandName}</h3>                        <p>{trimDescription(item?.brandDescription)}</p>                    </div>                ))}            </main>        </div>    );};