Python Script Revision for Image Capture

Need help fixing my python script.

Known Issues to fix.
1. Download all images including drawing as per format.
2. JSON output needs to reflect product details.
3. Should run through all products and eventually handle the collection pagination.



from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
import time
import pathlib
import requests
import re
import json

# === SETUP ===
DESKTOP = pathlib.Path.home() / "Desktop"
IMAGES = DESKTOP / "test_images"
PDFS = DESKTOP / "test_pdfs"
IMAGES.mkdir(exist_ok=True)
PDFS.mkdir(exist_ok=True)

options = Options()
options.add_argument("--window-size=1920,1080")
options.add_experimental_option("excludeSwitches", ["enable-automation"])

driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)

# === COLLECTION PAGE — FIRST 3 PRODUCTS ===
collection_url = "https://nerotapware.com.au/products?collection=bianca"
driver.get(collection_url)
time.sleep(15)

product_links = []
for a in driver.find_elements(By.CSS_SELECTOR, "a[href*='/products/bianca-']"):
href = a.get_attribute("href")
if href and href.startswith("https://nerotapware.com.au/products/bianca-"):
clean = href.split("#")[0].split("?")[0]
if clean not in product_links:
product_links.append(clean)
if len(product_links) >= 3:
break

print(f"Scraping {len(product_links)} products\n")

all_results = []

for start_url in product_links:
driver.get(start_url)
time.sleep(12)

title = driver.title.split("|")[0].strip()
print(f"→ {title}")

# === GET SKU ===
sku_match = re.search(r"NR\d+[A-Z]+", driver.page_source, re.I)
sku = sku_match.group().upper() if sku_match else "UNKNOWN"
sku_base = re.search(r"NR\d+", sku).group() if sku_match else sku

# === BUILD 6 FINISH URLs (100% correct) ===
current_clean = driver.current_url.split("#")[0].split("?")[0]
base_path = current_clean.rsplit("-", 2)[0] # removes finish + "only"

finishes = ["brushed-gold","chrome","matte-black","gun-metal","brushed-nickel","brushed-bronze"]
variant_urls = [f"{base_path}-{f}" for f in finishes]

# === JSON ===
product = {
"url": start_url,
"title": title,
"attributes": {
"SKU": sku,
"Collection": "Bianca",
"Finishes": ["Brushed Gold","Chrome","Matte Black","Gun Metal","Brushed Nickel","Brushed Bronze"],
"Material": "Brass",
"WELS Rating": "N/A",
"WELS Registration Number": "N/A",
"Availability": "In Stock"
},
"downloads": {},
"images": []
}

seen = set()
total = 0

# === VISIT EACH FINISH URL AND GRAB IMAGES ===
for v_url in variant_urls:
driver.get(v_url)
time.sleep(10) # Full load

finish = v_url.split("-")[-1].replace("gun-metal", "Gun Metal").title()
print(f" → {finish}")

for img in driver.find_elements(By.CSS_SELECTOR, ".product-gallery-media img"):
src = img.get_attribute("src")
if not src or src.startswith("data:"): continue

clean = src.split("?")[0]
name = clean.split("/")[-1]

if name in seen: continue
if not (name.lower().startswith(f"nr{sku.lower()[:5]}") or "bianca" in name.lower()): continue

try:
r = requests.get(clean, timeout=15)
if r.status_code == 200 and len(r.content) > 150000:
path = IMAGES / name
path.write_bytes(r.content)
product["images"].append(name)
seen.add(name)
total += 1
print(f" {name}")
except: pass

# === SPECIFICATION ===
spec_url = f"https://nero-tapware-assets.spicyweb.net.au/products/Products/{sku_base}.png"
spec_path = IMAGES / f"Specification_{sku_base}.png"
try:
r = requests.get(spec_url, timeout=15)
if r.status_code == 200:
spec_path.write_bytes(r.content)
product["downloads"]["specification"] = f"Specification_{sku_base}.png"
except: pass

# === PDFs ===
for a in driver.find_elements(By.TAG_NAME, "a"):
href = a.get_attribute("href") or ""
text = a.text.lower()
if not href.endswith(".pdf"): continue

if "install" in text:
name = f"{sku}_Installation.pdf"
path = PDFS / name
if not path.exists():
r = requests.get(href, timeout=20)
if r.status_code == 200:
path.write_bytes(r.content)
product["downloads"]["installation_pdf"] = name

if "spec" in text or "sheet" in text:
name = f"{sku}_Spec_Sheet.pdf"
path = PDFS / name
if not path.exists():
r = requests.get(href, timeout=20)
if r.status_code == 200:
path.write_bytes(r.content)
product["downloads"]["spec_sheet_pdf"] = name

all_results.append(product)
print(f" {total} total images\n")

driver.quit()

# === SAVE JSON ===
json_path = DESKTOP / "nero_bianca_final.json"
with open(json_path, "w", encoding="utf-8") as f:
json.dump(all_results, f, indent=2, ensure_ascii=False)

print(f"\nSUCCESS! {len(all_results)} products — ALL finishes")
print(f" Total images: {sum(len(p['images']) for p in all_results)}")
print(f" JSON: {json_path}")