대규모 데이터 수집을 위한 확장 가능한 비동기 웹 크롤러 구축

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

import aiohttp
import asyncio
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse

class AsyncCrawler:
    def __init__(self, max_concurrent=100, delay=1.0):
        self.max_concurrent = max_concurrent
        self.delay = delay
        self.session = None
        self.visited = set()
        self.queue = asyncio.Queue()

    async def fetch(self, url):
        """단일 URL을 비동기로 가져오기"""
        try:
            async with self.session.get(url, timeout=10) as response:
                if response.status == 200:
                    return await response.text()
                else:
                    print(f"Error {response.status}: {url}")
                    return None
        except asyncio.TimeoutError:
            print(f"Timeout: {url}")
            return None
        except Exception as e:
            print(f"Error fetching {url}: {e}")
            return None

    async def parse(self, html, base_url):
        """HTML에서 모든 링크 추출"""
        soup = BeautifulSoup(html, 'html.parser')
        links = []

        for link in soup.find_all('a', href=True):
            href = link['href']
            absolute_url = urljoin(base_url, href)
            links.append(absolute_url)

        return links

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

from urllib.robotparser import RobotFileParser

class RobotsChecker:
    def __init__(self):
        self.parsers = {}

    async def can_fetch(self, url, user_agent='*'):
        """robots.txt에 따라 URL 크롤링 가능 여부 확인"""
        parsed = urlparse(url)
        robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"

        if robots_url not in self.parsers:
            parser = RobotFileParser()
            parser.set_url(robots_url)
            try:
                parser.read()
                self.parsers[robots_url] = parser
            except:
                # robots.txt가 없으면 크롤링 허용
                return True

        return self.parsers[robots_url].can_fetch(user_agent, url)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

import time

class RateLimiter:
    def __init__(self, requests_per_second=10):
        self.delay = 1.0 / requests_per_second
        self.last_request = {}

    async def wait(self, domain):
        """도메인별 속도 제한 적용"""
        now = time.time()

        if domain in self.last_request:
            elapsed = now - self.last_request[domain]
            if elapsed < self.delay:
                await asyncio.sleep(self.delay - elapsed)

        self.last_request[domain] = time.time()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

from playwright.async_api import async_playwright

async def fetch_with_js(url):
    """JavaScript 렌더링이 포함된 URL 가져오기"""
    async with async_playwright() as p:
        browser = await p.chromium.launch()
        page = await browser.new_page()

        try:
            await page.goto(url, wait_until='networkidle')
            content = await page.content()
            return content
        finally:
            await browser.close()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

import sqlite3

class CrawlDatabase:
    def __init__(self, db_path='crawler.db'):
        self.conn = sqlite3.connect(db_path)
        self.create_tables()

    def create_tables(self):
        """데이터베이스 스키마 생성"""
        self.conn.execute('''
            CREATE TABLE IF NOT EXISTS urls (
                id INTEGER PRIMARY KEY,
                url TEXT UNIQUE,
                status INTEGER,
                content_type TEXT,
                crawled_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        ''')

        self.conn.execute('''
            CREATE TABLE IF NOT EXISTS links (
                source_url TEXT,
                target_url TEXT,
                anchor_text TEXT,
                FOREIGN KEY (source_url) REFERENCES urls(url)
            )
        ''')

        self.conn.commit()

    def add_url(self, url, status, content_type):
        """크롤링한 URL 저장"""
        try:
            self.conn.execute(
                'INSERT INTO urls (url, status, content_type) VALUES (?, ?, ?)',
                (url, status, content_type)
            )
            self.conn.commit()
        except sqlite3.IntegrityError:
            # URL이 이미 존재
            pass

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

async def crawl_website(start_url, max_pages=1000):
    """메인 크롤링 파이프라인"""
    crawler = AsyncCrawler(max_concurrent=50)
    robots = RobotsChecker()
    limiter = RateLimiter(requests_per_second=5)
    db = CrawlDatabase()

    # 세션 초기화
    async with aiohttp.ClientSession() as session:
        crawler.session = session
        await crawler.queue.put(start_url)

        pages_crawled = 0

        while not crawler.queue.empty() and pages_crawled < max_pages:
            url = await crawler.queue.get()

            # 방문 여부 확인
            if url in crawler.visited:
                continue

            # robots.txt 확인
            if not await robots.can_fetch(url):
                print(f"Blocked by robots.txt: {url}")
                continue

            # 속도 제한
            domain = urlparse(url).netloc
            await limiter.wait(domain)

            # 페치 및 파싱
            html = await crawler.fetch(url)
            if html:
                crawler.visited.add(url)
                pages_crawled += 1

                # 링크 추출
                links = await crawler.parse(html, url)

                # 새 링크를 큐에 추가
                for link in links:
                    if link not in crawler.visited:
                        await crawler.queue.put(link)

                # 데이터베이스에 저장
                db.add_url(url, 200, 'text/html')

                print(f"Crawled {pages_crawled}/{max_pages}: {url}")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

class DiskQueue:
    def __init__(self, db_path='queue.db'):
        self.conn = sqlite3.connect(db_path)
        self.conn.execute('''
            CREATE TABLE IF NOT EXISTS queue (
                url TEXT PRIMARY KEY,
                priority INTEGER DEFAULT 0,
                added_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        ''')

    async def put(self, url, priority=0):
        self.conn.execute(
            'INSERT OR IGNORE INTO queue (url, priority) VALUES (?, ?)',
            (url, priority)
        )
        self.conn.commit()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

from urllib.parse import urlparse, urlunparse

def normalize_url(url):
    """중복 방지를 위한 URL 정규화"""
    parsed = urlparse(url)

    # HTTPS 강제
    scheme = 'https'

    # www 접두사 제거
    netloc = parsed.netloc.replace('www.', '')

    # 후행 슬래시 제거
    path = parsed.path.rstrip('/')

    # 기본 포트 제거
    netloc = netloc.replace(':80', '').replace(':443', '')

    # 재구성
    return urlunparse((scheme, netloc, path, '', parsed.query, ''))