LoveTech Bots

LoveTech Amazon Bots

Summary

Selenium WebDriver is a great way to crawl almost any website!

Install Selenium WebDriver

Download Eclipse: https://www.eclipse.org/downloads/packages/eclipse-ide-java-developers/mars2

Download Java jdk: http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html

Download ChromeDriver.exe: https://sites.google.com/a/chromium.org/chromedriver/downloads

Download Selenium Java jar: http://www.seleniumhq.org/download/

Download JSON jar: https://mvnrepository.com/artifact/org.json/json/20170516

Common Bot Methods

These common methods are used by many of our bots:

public static String runJavascript(WebDriver driver, String script) {
	WebElement body = driver.findElement(By.cssSelector("body"));
	return (String)((JavascriptExecutor)driver).executeScript(script, body);
}

Amazon Bot:

import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintWriter;

import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;

public class AmazonBot {
	public static void main(String[] args) throws FileNotFoundException, InterruptedException {
		System.setProperty("webdriver.chrome.driver", "C:\\Users\\admin\\eclipse-workspace\\chromedriver.exe");
		WebDriver driver = new ChromeDriver();
		String page = "https://www.amazon.com/Intel-Desktop-Processor-i7-7700K-BX80677I77700K/dp/B01MXSI216";
		crawlProductPage(driver, page);
	}
	
	public static boolean crawlProductPage(WebDriver driver, String page) throws FileNotFoundException, InterruptedException {
		driver.get(page);
		loadjQuery(driver);
		double productPrice = runJavascriptDouble(driver, "var priceEl = $('#priceblock_ourprice'); if(priceEl.length == 0) { return -1; } return parseFloat(priceEl.text().replace(\"$\", \"\"));");
		Thread.sleep(330);
		double productRating = runJavascriptDouble(driver, "var rating = 0; var cell = jQuery('#averageCustomerReviews .a-icon-star .a-icon-alt'); if(cell.length > 0) { var val = parseFloat(cell.text().replace(\" out of 5 stars\", \"\")); if(!isNaN(val)) { rating = val}  } return rating;");
		String productImages = runJavascript(driver, "jQuery('.imageThumbnail .a-button-input').click(); var images = []; var imgs = jQuery('span[data-action=\"main-image-click\"] img'); for(var i = 0; i < imgs.length; i++) { images.push(jQuery(imgs[i]).attr('src')); } return images.join(', ');");
		return true;
	}
}

Explore Chakra7 Today


Explore Chakra7