From f05572e1242dcb908274e2c105372a112febcc23 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 23 Sep 2025 23:37:57 +0000
Subject: [PATCH 1/2] Initial plan
From 34790bf1a52c7f5fc939bf6dfa9d624c14f9c7ec Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 23 Sep 2025 23:49:48 +0000
Subject: [PATCH 2/2] Complete Trusted Shops web scraper implementation
Co-authored-by: blankspatrick1-cloud <225913654+blankspatrick1-cloud@users.noreply.github.com>
---
.gitignore | 3 +
SCRAPER_README.md | 97 +++++++++++++
requirements.txt | 3 +
scraper.py | 306 +++++++++++++++++++++++++++++++++++++++
scraper_demo.py | 356 ++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 765 insertions(+)
create mode 100644 SCRAPER_README.md
create mode 100644 scraper.py
create mode 100644 scraper_demo.py
diff --git a/.gitignore b/.gitignore
index e05e2e7..117f6f2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -159,3 +159,6 @@ cython_debug/
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
.DS_Store
+
+# Scraper output files
+shops_*.csv
diff --git a/SCRAPER_README.md b/SCRAPER_README.md
new file mode 100644
index 0000000..93328f3
--- /dev/null
+++ b/SCRAPER_README.md
@@ -0,0 +1,97 @@
+# Trusted Shops Web Scraper
+
+A comprehensive web scraping tool that extracts company information from the Trusted Shops website (https://www.trustedshops.de).
+
+## Features
+
+- **Pagination Handling**: Automatically processes multiple pages by incrementing the page parameter
+- **Comprehensive Data Extraction**: Collects the following information for each company:
+ - Company Name
+ - Logo URL
+ - Profile URL
+ - Company Website URL
+ - Phone Number
+ - Physical Address
+ - Business Categories/Tags
+ - Email Address
+ - Company Description
+
+- **CSV Output**: Saves data to a timestamped CSV file (e.g., `shops_2025-09-23_23-42-14.csv`)
+- **Incremental Saving**: Data is saved after each profile is processed to prevent data loss
+- **Error Handling**: Includes retry logic and graceful error handling
+- **Rate Limiting**: Built-in delays between requests to respect server resources
+
+## Files
+
+- `scraper.py` - Main scraping script for production use
+- `scraper_demo.py` - Demo version with mock data for testing
+- `requirements.txt` - Updated with web scraping dependencies
+
+## Installation
+
+1. Install the required dependencies:
+```bash
+pip install -r requirements.txt
+```
+
+## Usage
+
+### Production Scraper
+
+Run the main scraper (requires internet access):
+```bash
+python scraper.py
+```
+
+### Demo Version
+
+Test the functionality with mock data:
+```bash
+python scraper_demo.py
+```
+
+## Output Format
+
+The scraper creates a CSV file with the following columns:
+
+| Column | Description |
+|--------|-------------|
+| Company Name | Name of the business |
+| Logo | URL to company logo image |
+| Profile URL | Link to the Trusted Shops profile page |
+| Company URL | Company's official website |
+| Phone | Contact phone number |
+| Address | Physical business address |
+| Tags | Business categories/tags |
+| Email | Contact email address |
+| Description | Company description/overview |
+
+## Configuration
+
+The scraper can be configured by modifying the `TrustedShopsScraper` class:
+
+- `base_url`: Target URL for scraping (default: computer/electronics category)
+- Request delays: Modify `time.sleep()` values to adjust scraping speed
+- Retry logic: Adjust `max_retries` parameter in `get_page()` method
+
+## Technical Details
+
+- **Framework**: Python 3.x
+- **Libraries**: BeautifulSoup4, requests, pandas, re
+- **Approach**: Sequential page processing with profile detail extraction
+- **Error Recovery**: Retry mechanism for failed requests
+- **Data Persistence**: Incremental CSV writing
+
+## Notes
+
+- The scraper includes proper delays between requests to be respectful to the target server
+- All extracted data is cleaned and formatted for consistency
+- The script handles various HTML structures and missing data gracefully
+- BeautifulSoup warnings have been addressed using current best practices
+
+## Example Output
+
+```csv
+Company Name,Logo,Profile URL,Company URL,Phone,Address,Tags,Email,Description
+EnjoyYourCamera.com,https://channel-settings.etrusted.com/logo-932f448d...,https://www.trustedshops.de/bewertung/info_X233BF...,https://www.enjoyyourcamera.com,+49 511 20029090,"ENJOYYOURBRANDS GmbH, Eleonorenstr. 20, Deutschland","Bücher, Computer, Unterhaltungselektronik & Zubehör",shop@enjoyyourcamera.com,"Enjoyyourcamera.com ist Ihr Versandhaus für Spezial-Fotozubehör..."
+```
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index ab6c294..10e9fc4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,7 @@
aiofiles==23.1.0
annotated-types==0.5.0
anyio==3.7.1
+beautifulsoup4>=4.12.0
Brotli==1.0.9
certifi==2023.7.22
click==8.1.6
@@ -14,8 +15,10 @@ httpx==0.24.1
hyperframe==6.0.1
idna==3.4
lxml==4.9.3
+pandas>=2.0.0
pydantic==2.1.1
pydantic_core==2.4.0
+requests>=2.28.0
sniffio==1.3.0
socksio==1.0.0
starlette==0.27.0
diff --git a/scraper.py b/scraper.py
new file mode 100644
index 0000000..fed5291
--- /dev/null
+++ b/scraper.py
@@ -0,0 +1,306 @@
+#!/usr/bin/env python3
+"""
+Trusted Shops Web Scraper
+
+This script scrapes company information from Trusted Shops website including:
+- Company name, logo, profile URL from listing pages
+- Additional details from profile pages: company URL, address, tags, email, description
+
+Features:
+- Pagination handling
+- Timestamped CSV output
+- Error handling and delays between requests
+- Incremental saving to prevent data loss
+"""
+
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+import time
+import re
+from datetime import datetime
+import csv
+import os
+
+
+class TrustedShopsScraper:
+ def __init__(self, base_url="https://www.trustedshops.de/shops/computer_unterhaltungselektronik_zubehor/"):
+ self.base_url = base_url
+ self.session = requests.Session()
+ self.session.headers.update({
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+ })
+
+ # Create timestamped filename
+ timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+ self.csv_filename = f"shops_{timestamp}.csv"
+
+ # CSV headers
+ self.headers = [
+ 'Company Name', 'Logo', 'Profile URL', 'Company URL',
+ 'Phone', 'Address', 'Tags', 'Email', 'Description'
+ ]
+
+ # Initialize CSV file with headers
+ self._init_csv()
+
+ def _init_csv(self):
+ """Initialize CSV file with headers if it doesn't exist"""
+ if not os.path.exists(self.csv_filename):
+ with open(self.csv_filename, 'w', newline='', encoding='utf-8') as file:
+ writer = csv.writer(file)
+ writer.writerow(self.headers)
+ print(f"Created CSV file: {self.csv_filename}")
+
+ def _save_to_csv(self, data):
+ """Append data to CSV file"""
+ with open(self.csv_filename, 'a', newline='', encoding='utf-8') as file:
+ writer = csv.writer(file)
+ writer.writerow(data)
+
+ def get_page(self, url, max_retries=3):
+ """Get page content with retry logic"""
+ for attempt in range(max_retries):
+ try:
+ response = self.session.get(url, timeout=10)
+ response.raise_for_status()
+ return response
+ except requests.exceptions.RequestException as e:
+ print(f"Attempt {attempt + 1} failed for {url}: {e}")
+ if attempt < max_retries - 1:
+ time.sleep(2 ** attempt) # Exponential backoff
+ else:
+ print(f"Failed to fetch {url} after {max_retries} attempts")
+ return None
+
+ def extract_shops_from_page(self, page_content):
+ """Extract shop information from a listing page"""
+ soup = BeautifulSoup(page_content, 'html.parser')
+ shops = []
+
+ # Find all shop entries - adjust selectors based on actual HTML structure
+ shop_elements = soup.find_all('div', class_=re.compile(r'shop|item|card|listing'))
+
+ if not shop_elements:
+ # Try alternative selectors
+ shop_elements = soup.find_all('a', href=re.compile(r'/bewertung/info_'))
+
+ print(f"Found {len(shop_elements)} potential shop elements")
+
+ for element in shop_elements:
+ try:
+ # Extract company name
+ name_elem = element.find(['h2', 'h3', 'h4', 'span', 'div'], class_=re.compile(r'name|title|company'))
+ if not name_elem:
+ name_elem = element.find('a', href=re.compile(r'/bewertung/info_'))
+
+ company_name = name_elem.get_text(strip=True) if name_elem else "N/A"
+
+ # Extract logo URL
+ logo_elem = element.find('img')
+ logo_url = logo_elem.get('src', '') if logo_elem else "N/A"
+ if logo_url and logo_url.startswith('//'):
+ logo_url = 'https:' + logo_url
+ elif logo_url and logo_url.startswith('/'):
+ logo_url = 'https://www.trustedshops.de' + logo_url
+
+ # Extract profile URL
+ profile_link = element.find('a', href=re.compile(r'/bewertung/info_'))
+ if not profile_link and element.name == 'a':
+ profile_link = element
+
+ profile_url = ""
+ if profile_link:
+ href = profile_link.get('href', '')
+ if href.startswith('/'):
+ profile_url = 'https://www.trustedshops.de' + href
+ else:
+ profile_url = href
+
+ if company_name != "N/A" and profile_url:
+ shops.append({
+ 'company_name': company_name,
+ 'logo_url': logo_url,
+ 'profile_url': profile_url
+ })
+ print(f"Extracted: {company_name[:50]}...")
+
+ except Exception as e:
+ print(f"Error extracting shop data: {e}")
+ continue
+
+ return shops
+
+ def extract_profile_details(self, profile_url):
+ """Extract additional details from profile page"""
+ print(f"Fetching profile: {profile_url}")
+
+ response = self.get_page(profile_url)
+ if not response:
+ return {
+ 'company_url': 'N/A',
+ 'phone': 'N/A',
+ 'address': 'N/A',
+ 'tags': 'N/A',
+ 'email': 'N/A',
+ 'description': 'N/A'
+ }
+
+ soup = BeautifulSoup(response.content, 'html.parser')
+
+ # Extract company URL
+ company_url = "N/A"
+ url_links = soup.find_all('a', href=True)
+ for link in url_links:
+ href = link.get('href', '')
+ if any(domain in href for domain in ['.com', '.de', '.org', '.net']) and 'trustedshops' not in href:
+ company_url = href
+ break
+
+ # Extract contact information from address block
+ phone = "N/A"
+ address = "N/A"
+ email = "N/A"
+
+ # Look for contact section or address information
+ contact_section = soup.find(string=re.compile(r'Kontakt|Adresse|Address'))
+ if contact_section:
+ # Get parent element and extract text
+ contact_parent = contact_section.parent
+ if contact_parent:
+ contact_text = contact_parent.get_text()
+
+ # Extract phone using regex
+ phone_match = re.search(r'\+?\d{1,4}[\s\-]?\d{1,4}[\s\-]?\d{4,}', contact_text)
+ if phone_match:
+ phone = phone_match.group().strip()
+
+ # Extract email using regex
+ email_match = re.search(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', contact_text)
+ if email_match:
+ email = email_match.group().strip()
+
+ # Clean address by removing phone and email
+ address = contact_text
+ if phone != "N/A":
+ address = address.replace(phone, '').strip()
+ if email != "N/A":
+ address = address.replace(email, '').strip()
+ if company_url != "N/A":
+ address = address.replace(company_url, '').strip()
+
+ # Clean up extra whitespace and newlines
+ address = re.sub(r'\s+', ' ', address).strip()
+
+ # Also check for mailto links for email
+ if email == "N/A":
+ mailto_link = soup.find('a', href=re.compile(r'mailto:'))
+ if mailto_link:
+ email = mailto_link.get('href', '').replace('mailto:', '')
+
+ # Extract tags/categories
+ tags = "N/A"
+ categories_section = soup.find(string=re.compile(r'Kategorien|Categories'))
+ if categories_section:
+ categories_parent = categories_section.find_parent()
+ if categories_parent:
+ category_links = categories_parent.find_all('a')
+ if category_links:
+ tags = ', '.join([link.get_text(strip=True) for link in category_links])
+
+ # Extract description
+ description = "N/A"
+ # Look for description or company info
+ desc_keywords = ['Beschreibung', 'Description', 'Über uns', 'About', 'Unternehmen']
+ for keyword in desc_keywords:
+ desc_section = soup.find(string=re.compile(keyword))
+ if desc_section:
+ desc_parent = desc_section.find_parent()
+ if desc_parent:
+ # Get next sibling or content within the same element
+ desc_text = desc_parent.get_text(strip=True)
+ if len(desc_text) > len(keyword) + 10: # Ensure it's not just the keyword
+ description = desc_text
+ break
+
+ # If no description found, try to get meta description
+ if description == "N/A":
+ meta_desc = soup.find('meta', attrs={'name': 'description'})
+ if meta_desc:
+ description = meta_desc.get('content', 'N/A')
+
+ return {
+ 'company_url': company_url,
+ 'phone': phone,
+ 'address': address,
+ 'tags': tags,
+ 'email': email,
+ 'description': description
+ }
+
+ def scrape_all_pages(self):
+ """Main scraping function that handles pagination"""
+ page = 1
+ total_shops = 0
+
+ while True:
+ print(f"\nScraping page {page}...")
+ url = f"{self.base_url}?page={page}"
+
+ response = self.get_page(url)
+ if not response:
+ print(f"Failed to fetch page {page}")
+ break
+
+ shops = self.extract_shops_from_page(response.content)
+
+ if not shops:
+ print(f"No shops found on page {page}. Ending scraping.")
+ break
+
+ print(f"Found {len(shops)} shops on page {page}")
+
+ # Process each shop
+ for i, shop in enumerate(shops, 1):
+ print(f"Processing shop {i}/{len(shops)}: {shop['company_name'][:50]}...")
+
+ # Get additional details from profile page
+ profile_details = self.extract_profile_details(shop['profile_url'])
+
+ # Combine all data
+ row_data = [
+ shop['company_name'],
+ shop['logo_url'],
+ shop['profile_url'],
+ profile_details['company_url'],
+ profile_details['phone'],
+ profile_details['address'],
+ profile_details['tags'],
+ profile_details['email'],
+ profile_details['description']
+ ]
+
+ # Save to CSV immediately
+ self._save_to_csv(row_data)
+ total_shops += 1
+
+ # Add delay to be respectful to the server
+ time.sleep(2)
+
+ print(f"Completed page {page}. Total shops processed: {total_shops}")
+ page += 1
+
+ # Add delay between pages
+ time.sleep(1)
+
+ print(f"\nScraping completed! Total shops scraped: {total_shops}")
+ print(f"Results saved to: {self.csv_filename}")
+
+
+def main():
+ scraper = TrustedShopsScraper()
+ scraper.scrape_all_pages()
+
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/scraper_demo.py b/scraper_demo.py
new file mode 100644
index 0000000..9b451b3
--- /dev/null
+++ b/scraper_demo.py
@@ -0,0 +1,356 @@
+#!/usr/bin/env python3
+"""
+Demo/Test version of Trusted Shops Web Scraper
+
+This is a demonstration version that works with mock HTML data to show
+how the scraper would work in a real environment with internet access.
+"""
+
+import csv
+import os
+import re
+from datetime import datetime
+from bs4 import BeautifulSoup
+
+
+class MockTrustedShopsScraper:
+ def __init__(self):
+ # Create timestamped filename
+ timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+ self.csv_filename = f"shops_demo_{timestamp}.csv"
+
+ # CSV headers
+ self.headers = [
+ 'Company Name', 'Logo', 'Profile URL', 'Company URL',
+ 'Phone', 'Address', 'Tags', 'Email', 'Description'
+ ]
+
+ # Initialize CSV file with headers
+ self._init_csv()
+
+ def _init_csv(self):
+ """Initialize CSV file with headers if it doesn't exist"""
+ if not os.path.exists(self.csv_filename):
+ with open(self.csv_filename, 'w', newline='', encoding='utf-8') as file:
+ writer = csv.writer(file)
+ writer.writerow(self.headers)
+ print(f"Created CSV file: {self.csv_filename}")
+
+ def _save_to_csv(self, data):
+ """Append data to CSV file"""
+ with open(self.csv_filename, 'a', newline='', encoding='utf-8') as file:
+ writer = csv.writer(file)
+ writer.writerow(data)
+
+ def get_mock_listing_page(self):
+ """Return mock HTML for a Trusted Shops listing page"""
+ return '''
+
+
+
+
+
+
+
+
+
+
+
+
+
Enjoyyourcamera.com ist Ihr Versandhaus für Spezial-Fotozubehör, Kamerazubehör, Studio-Zubehör. Hier finden Sie zum Beispiel Fototaschen, Akkus und Batterien, Digitalkamerazubehör, Blitzgeräte, Filter, Stative, Objektive und Speichermedien von den Marken JJC, KT, Marumi, Matin, Mennon, Nissin, Ownuser, Pedco, Seculine und VisibleDust.
+
+
TechStore24.de - Ihr zuverlässiger Partner für Computer, Laptops, Smartphones und Elektronikzubehör. Wir bieten qualitativ hochwertige Produkte zu fairen Preisen.
+
+
ComputerWorld GmbH ist spezialisiert auf Business-IT-Lösungen, Hardware-Verkauf und IT-Services für Unternehmen jeder Größe.
+
tags in contact section
+ contact_paragraphs = contact_div.find_all('p')
+ address_lines = []
+ for p in contact_paragraphs:
+ p_text = p.get_text(strip=True)
+ # Skip phone, email, and website lines
+ if (not re.match(r'\+?\d', p_text) and
+ '@' not in p_text and
+ 'www.' not in p_text and
+ 'http' not in p_text and
+ len(p_text) > 5): # Avoid empty or very short lines
+
+ # Split by
tags if they exist
+ if p.find('br'):
+ lines = p_text.split('\n') if '\n' in p_text else [p_text]
+ for line in lines:
+ line = line.strip()
+ if line:
+ address_lines.append(line)
+ else:
+ address_lines.append(p_text)
+
+ if address_lines:
+ address = ', '.join(address_lines)
+
+ # Extract tags/categories
+ tags = "N/A"
+ categories_div = soup.find('div', class_='categories')
+ if categories_div:
+ category_links = categories_div.find_all('a')
+ if category_links:
+ tags = ', '.join([link.get_text(strip=True) for link in category_links])
+
+ # Extract description
+ description = "N/A"
+ desc_elem = soup.find('div', class_='description')
+ if desc_elem:
+ desc_p = desc_elem.find('p')
+ if desc_p:
+ description = desc_p.get_text(strip=True)
+
+ return {
+ 'company_url': company_url,
+ 'phone': phone,
+ 'address': address,
+ 'tags': tags,
+ 'email': email,
+ 'description': description
+ }
+
+ def run_demo_scrape(self):
+ """Run demo scraping with mock data"""
+ print("Running demo scrape with mock data...")
+ print("\nScraping page 1...")
+
+ # Get mock listing page
+ page_content = self.get_mock_listing_page()
+ shops = self.extract_shops_from_page(page_content)
+
+ if not shops:
+ print("No shops found in demo data")
+ return
+
+ print(f"Found {len(shops)} shops in demo data")
+
+ # Process each shop
+ for i, shop in enumerate(shops, 1):
+ print(f"\nProcessing shop {i}/{len(shops)}: {shop['company_name']}")
+
+ # Get additional details from profile page
+ profile_details = self.extract_profile_details(shop['profile_url'], shop['company_name'])
+
+ # Combine all data
+ row_data = [
+ shop['company_name'],
+ shop['logo_url'],
+ shop['profile_url'],
+ profile_details['company_url'],
+ profile_details['phone'],
+ profile_details['address'],
+ profile_details['tags'],
+ profile_details['email'],
+ profile_details['description']
+ ]
+
+ # Save to CSV
+ self._save_to_csv(row_data)
+ print(f"Saved data for {shop['company_name']}")
+
+ print(f"\nDemo scraping completed! Total shops processed: {len(shops)}")
+ print(f"Results saved to: {self.csv_filename}")
+
+ # Display the CSV content
+ self.display_csv_content()
+
+ def display_csv_content(self):
+ """Display the CSV content for verification"""
+ print(f"\n--- Content of {self.csv_filename} ---")
+ try:
+ with open(self.csv_filename, 'r', encoding='utf-8') as file:
+ reader = csv.reader(file)
+ for i, row in enumerate(reader):
+ if i == 0: # Header row
+ print("Headers:", " | ".join(row))
+ print("-" * 100)
+ else:
+ print(f"Row {i}:")
+ for j, (header, value) in enumerate(zip(self.headers, row)):
+ print(f" {header}: {value[:100]}{'...' if len(value) > 100 else ''}")
+ print("-" * 50)
+ except Exception as e:
+ print(f"Error reading CSV: {e}")
+
+
+def main():
+ scraper = MockTrustedShopsScraper()
+ scraper.run_demo_scrape()
+
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file