Building a Price Monitoring Service with Web Scraping

Price monitoring is one of the most commercially viable applications of web scraping. E-commerce businesses, retailers, and resellers all need to track competitor prices. Here is how to build a price monitoring service.

Core Architecture

Scheduler → Scraping Queue → ScraperAPI → Parser → Database → Alert Engine → Dashboard
  (cron)      (Redis/SQS)                                        (Email/Slack)

Step 1: Price Scraper

import requests
from bs4 import BeautifulSoup
import re
from datetime import datetime

SCRAPERAPI_KEY = "YOUR_SCRAPERAPI_KEY"

def scrape_price(url):
    """Scrape the current price from a product URL."""
    response = requests.get(
        "http://api.scraperapi.com",
        params={"api_key": SCRAPERAPI_KEY, "url": url}
    )

    soup = BeautifulSoup(response.text, "html.parser")

    # Try JSON-LD first (most reliable)
    import json
    ld_script = soup.find("script", type="application/ld+json")
    if ld_script:
        try:
            data = json.loads(ld_script.string)
            if "offers" in data:
                offers = data["offers"]
                if isinstance(offers, list):
                    offers = offers[0]
                return {
                    "price": float(offers.get("price", 0)),
                    "currency": offers.get("priceCurrency", "USD"),
                    "availability": offers.get("availability", ""),
                    "scraped_at": datetime.utcnow().isoformat()
                }
        except (json.JSONDecodeError, ValueError):
            pass

    # Fallback: look for common price patterns
    price_patterns = [
        soup.find("meta", {"property": "product:price:amount"}),
        soup.find(attrs={"data-price": True}),
        soup.find(class_=re.compile(r"price", re.I))
    ]

    for element in price_patterns:
        if element:
            price_text = element.get("content") or element.get("data-price") or element.get_text()
            price_match = re.search(r"[\d,.]+", price_text)
            if price_match:
                return {
                    "price": float(price_match.group().replace(",", "")),
                    "currency": "USD",
                    "scraped_at": datetime.utcnow().isoformat()
                }

    return None

Step 2: Price Database

import sqlite3

def init_db():
    conn = sqlite3.connect("prices.db")
    conn.execute("""
        CREATE TABLE IF NOT EXISTS price_history (
            id INTEGER PRIMARY KEY,
            product_url TEXT,
            product_name TEXT,
            price REAL,
            currency TEXT,
            scraped_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
        )
    """)
    conn.execute("""
        CREATE TABLE IF NOT EXISTS alerts (
            id INTEGER PRIMARY KEY,
            product_url TEXT,
            alert_type TEXT,
            threshold REAL,
            email TEXT,
            active BOOLEAN DEFAULT 1
        )
    """)
    return conn

def save_price(conn, url, name, price, currency):
    conn.execute(
        "INSERT INTO price_history (product_url, product_name, price, currency) VALUES (?, ?, ?, ?)",
        (url, name, price, currency)
    )
    conn.commit()

Step 3: Alert System

def check_price_alerts(conn, url, current_price):
    """Check if any alert thresholds have been triggered."""
    alerts = conn.execute(
        "SELECT * FROM alerts WHERE product_url = ? AND active = 1",
        (url,)
    ).fetchall()

    for alert in alerts:
        alert_type = alert[2]
        threshold = alert[3]
        email = alert[4]

        triggered = False
        if alert_type == "below" and current_price < threshold:
            triggered = True
            message = f"Price dropped to {current_price} (below {threshold})"
        elif alert_type == "change":
            # Get previous price
            prev = conn.execute(
                "SELECT price FROM price_history WHERE product_url = ? ORDER BY scraped_at DESC LIMIT 1 OFFSET 1",
                (url,)
            ).fetchone()
            if prev:
                change_pct = abs(current_price - prev[0]) / prev[0] * 100
                if change_pct > threshold:
                    triggered = True
                    message = f"Price changed by {change_pct:.1f}%"

        if triggered:
            send_alert(email, message)

Monetization

Price monitoring services typically charge:

• $29-99/month for small businesses (50-500 products)
• $199-499/month for mid-market (500-5,000 products)
• $999+/month for enterprise (unlimited products, API access, custom integrations)

The key differentiator is data accuracy and speed of detection. Using ScraperAPI ensures reliable scraping across protected e-commerce sites, which is critical for a monitoring service customers depend on.