Overview:
This script uses Puppeteer (a headless Chrome Node.js API) to simulate a real user browsing a website through a specified proxy server. It fetches a random desktop user agent from Trusted Proxies’ online list and assigns it to the browser session. The target URL is set to https://httpbin.org/ip, which returns the visitor’s IP address in JSON format, allowing you to verify that your proxy is functioning correctly.
Key Features:
Proxy Setup: The script retrieves the proxy address from a command-line flag using "--proxy-server=http://your-proxy:port". This ensures that all browsing is routed through the provided proxy.
Dynamic User Agent: A random user agent is fetched from "https://customers.trustedproxies.com/downloads/desktop_useragents.txt", ensuring that each session can emulate a different desktop browser.
Headless or Headed Sessions: The script is set to run in headed mode (headless: false), meaning the browser window is visible.
To run in headless mode, change the headless option to true.
Slow Scrolling: The script scrolls down the page slowly with a 500ms interval between each scroll step, mimicking natural human scrolling behavior.
Target URL: The URL https://httpbin.org/ip is used to display the IP address currently visible to the server, helping confirm that the proxy is properly applied.
Usage Instructions:
Install Dependencies: Run the following command in your project directory: npm install puppeteer node-fetch@2
Specify Your Proxy: When executing the script, supply your proxy server using the "--proxy-server" flag.
Example: node kb-script.js --proxy-server=http://shp-testuser-us-v00001.tp-ns.com:80
Choose Session Mode: To run with a visible browser window (headed mode), ensure that headless is set to false. For a headless session, set headless to true in the puppeteer.launch options.
Run the Script: The script will launch Chrome with the provided proxy settings, apply a random user agent, navigate to https://httpbin.org/ip, scroll down the page slowly, and then close the browser after a brief delay.
This documentation should help users understand how the script works and how to configure it to test their proxy server setup using Puppeteer.
Code:
const puppeteer = require('puppeteer');
const fetch = require('node-fetch'); // npm install node-fetch@2
const process = require('process');
// Helper function for delay
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
// Retrieve proxy server from command-line arguments (e.g., --proxy-server=http://your-proxy:port)
const args = process.argv.slice(2);
let proxyServer = null;
args.forEach(arg => {
if (arg.startsWith('--proxy-server=')) {
proxyServer = arg.split('=')[1];
}
});
if (!proxyServer) {
console.error("Error: Please provide a proxy using the --proxy-server flag. Example: --proxy-server=http://your-proxy:port");
process.exit(1);
}
// Function to fetch a random user agent from Trusted Proxies
async function getRandomUserAgent() {
try {
const response = await fetch("https://customers.trustedproxies.com/downloads/desktop_useragents.txt", { timeout: 10000 });
if (!response.ok) {
throw new Error(`Failed to fetch user agents: ${response.statusText}`);
}
const text = await response.text();
const userAgents = text.split(/\r?\n/).filter(ua => ua.trim().length);
// If no user agents are found, throw an error instead of defaulting
if (userAgents.length === 0) {
throw new Error("No user agents found in the fetched data.");
}
const randomIndex = Math.floor(Math.random() * userAgents.length);
return userAgents[randomIndex];
} catch (e) {
console.error("Error fetching user agents:", e);
throw new Error("Error fetching user agents.");
}
}
// Define a list of sample URLs to browse
const urls = [
'https://httpbin.org/ip'
];
const randomUrl = urls[Math.floor(Math.random() * urls.length)];
(async () => {
// Fetch a random user agent from the provided URL
const randomUserAgent = await getRandomUserAgent();
// Launch Puppeteer with the specified proxy-server and Chrome options
const browser = await puppeteer.launch({
headless: false,
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
`--proxy-server=${proxyServer}`,
'--incognito',
'--window-size=1920,1080',
'--disable-infobars',
'--disable-blink-features=AutomationControlled',
'--disable-extensions',
'--disable-dev-shm-usage',
'--disable-gpu',
'--log-level=3'
]
});
const page = await browser.newPage();
await page.setUserAgent(randomUserAgent);
await page.setViewport({ width: 1920, height: 1080 });
// Remove the navigator.webdriver property to minimize detection
await page.evaluateOnNewDocument(() => {
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined,
});
});
console.log(`Using proxy-server: ${proxyServer}`);
console.log(`Using User-Agent: ${randomUserAgent}`);
console.log(`Navigating to: ${randomUrl}`);
try {
await page.goto(randomUrl, { waitUntil: 'networkidle2', timeout: 30000 });
// Scroll down the page gradually (using a slower interval of 500ms)
await page.evaluate(async () => {
await new Promise((resolve) => {
let totalHeight = 0;
const distance = 100;
const timer = setInterval(() => {
window.scrollBy(0, distance);
totalHeight += distance;
if (totalHeight >= document.body.scrollHeight) {
clearInterval(timer);
resolve();
}
}, 500);
});
});
// Wait for an additional 5 seconds before closing the browser
await delay(5000);
} catch (e) {
console.error("Error during navigation:", e);
}
await browser.close();
})();
