웹 스크래핑 기초: BeautifulSoup

# ⚠️ 이 코드는 허가된 환경에서만 사용하세요.
# pip install beautifulsoup4 lxml
from bs4 import BeautifulSoup
import requests

html = requests.get('https://example.com', timeout=5).text
soup = BeautifulSoup(html, 'lxml')  # lxml 파서가 가장 빠름

# 첫 번째 매칭 — find
title = soup.find('title')
print(title.text)  # "Example Domain"

# 모두 매칭 — find_all
links = soup.find_all('a')
for a in links:
    print(a.get('href'), '→', a.text)

# CSS 선택자 — select / select_one (jQuery 스타일)
nav_links = soup.select('nav a.menu-item')
first_p = soup.select_one('article > p:first-child')

# 모든 링크
hrefs = [a['href'] for a in soup.find_all('a', href=True)]

# 외부 링크만
import re
external = [h for h in hrefs if re.match(r'^https?://', h)]

# 이메일 추출
emails = re.findall(r'[\w.+-]+@[\w-]+\.[\w.-]+', html)

# 메타 태그
description = soup.find('meta', attrs={'name': 'description'})
if description:
    print(description.get('content'))

# 폼 필드 추출 (CSRF 토큰 찾기)
form = soup.find('form', id='login')
if form:
    csrf = form.find('input', attrs={'name': 'csrf_token'})
    if csrf:
        print('CSRF 토큰:', csrf['value'])

def extract_tables(html: str) -> list[list[list[str]]]:
    """모든 <table>의 행/열을 2D 배열로 추출."""
    soup = BeautifulSoup(html, 'lxml')
    tables = []
    for table in soup.find_all('table'):
        rows = []
        for tr in table.find_all('tr'):
            cells = [c.get_text(strip=True) for c in tr.find_all(['td', 'th'])]
            if cells:
                rows.append(cells)
        if rows:
            tables.append(rows)
    return tables

from urllib.parse import urljoin
from urllib.robotparser import RobotFileParser

def can_scrape(url: str, user_agent: str = 'WhiteHat/1.0') -> bool:
    """robots.txt 정책 확인 — 스크래핑 허용 여부."""
    rp = RobotFileParser()
    base = urljoin(url, '/robots.txt')
    rp.set_url(base)
    try:
        rp.read()
        return rp.can_fetch(user_agent, url)
    except Exception:
        return False  # 못 읽으면 보수적으로 거부

# 항상 먼저 체크
if can_scrape('https://example.com/page'):
    r = requests.get('https://example.com/page')

def page_recon(url: str) -> dict:
    """OSINT 정찰 — 공개 정보만 추출 (윤리적 범위)."""
    if not can_scrape(url):
        return {'error': 'robots.txt 거부 — 스크래핑 금지'}

    r = requests.get(url, timeout=5, headers={'User-Agent': 'WhiteHat/1.0'})
    soup = BeautifulSoup(r.text, 'lxml')

    # 1. 이메일 (정규식 + obfuscation 처리)
    text = soup.get_text()
    emails = set(re.findall(r'[\w.+-]+@[\w-]+\.[\w.-]+', text))
    # "name [at] domain [dot] com" 변형 처리
    obfuscated = re.findall(r'(\w+)\s*\[at\]\s*([\w-]+)\s*\[dot\]\s*(\w+)', text, re.I)
    for u, d, t in obfuscated:
        emails.add(f'{u}@{d}.{t}')

    # 2. 외부 도메인
    hosts = set()
    for a in soup.find_all('a', href=True):
        href = a['href']
        m = re.match(r'^https?://([^/]+)', href)
        if m:
            hosts.add(m.group(1))

    # 3. 폼 정보
    forms = []
    for form in soup.find_all('form'):
        forms.append({
            'action': form.get('action'),
            'method': form.get('method', 'GET').upper(),
            'inputs': [inp.get('name') for inp in form.find_all('input') if inp.get('name')],
        })

    # 4. 메타데이터
    meta = {}
    for tag in soup.find_all('meta'):
        name = tag.get('name') or tag.get('property')
        if name and tag.get('content'):
            meta[name] = tag['content']

    return {
        'url': r.url,
        'emails': sorted(emails),
        'external_hosts': sorted(hosts),
        'forms': forms,
        'meta': meta,
    }