[Dataset File]

크롤링 하는 법

• 코랩에서 크롬 및 크롬 드라이브 다운 받는 코드 (크롬과 크롬 드라이버의 버전을 잘 맞춰야 한다)

# Chrome 버전: 128
!wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
!dpkg -i google-chrome-stable_current_amd64.deb
!apt --fix-broken install -y

# ChromeDriver 다운로드 및 설치
!wget https://storage.googleapis.com/chrome-for-testing-public/128.0.6613.119/linux64/chromedriver-linux64.zip
!unzip chromedriver-linux64.zip
!chmod +x ./chromedriver-linux64/chromedriver

• 실수로 폴더 만들었을 때 코랩에서 지우는 방법

import shutil

# 'folder_name'을 삭제합니다.
shutil.rmtree('chromedriver-linux64')

• main 코드 (google image download 함수가 있기도 한데, 이건 잘 안먹힌다..) + 코랩에서는 GUI 없이 열어야 한다? → 코랩에서는 크롬이 아닌 크로미움이란 크롬의 개발자 버전을 사용해야 한다?

import os
import time
import urllib.request
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.options import Options

# 키워드와 폴더 이름 설정
keywords = [
    "amusement drawing",
    "awe drawing",
    "contentment drawing",
    "excitement drawing",
    "anger drawing",
    "disgust drawing",
    "fear drawing",
    "sadness drawing"
]

# 이미지 저장할 폴더 경로
output_dir = os.path.expanduser("emotion_images")  # 바탕화면 경로

# 크롤링할 이미지 개수
num_images = 200

# 크롬 옵션 설정 (헤드리스 모드 비활성화)
chrome_options = Options()
# chrome_options.add_argument("--headless")  # GUI를 표시하지 않음 (디버깅 시 비활성화)
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-dev-shm-usage")

# 크롬 드라이버 설정
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options)

def download_image(image_url, folder_path, image_num):
    try:
        print(f"Downloading image {image_num}: {image_url}")
        if image_url.startswith("http"):
            opener = urllib.request.build_opener()
            opener.addheaders = [('User-Agent', 'Mozilla/5.0')]
            urllib.request.install_opener(opener)
            file_name = f"{image_num}.jpg"
            file_path = os.path.join(folder_path, file_name)
            urllib.request.urlretrieve(image_url, file_path)
            print(f"Image {image_num} downloaded: {file_path}")
        else:
            print(f"Invalid URL for image {image_num}: {image_url}")
    except Exception as e:
        print(f"Error downloading image {image_num}: {e}")

def create_folder_if_not_exists(folder_name):
    if not os.path.exists(folder_name):
        os.makedirs(folder_name)

def crawl_images(keyword):
    print(f"Searching images for: {keyword}")
    search_url = f"https://www.google.com/search?q={keyword}&tbm=isch"
    driver.get(search_url)
    
    image_urls = set()
    last_height = driver.execute_script("return document.body.scrollHeight")
    
    while len(image_urls) < num_images:
        # 이미지 URL 추출
        images = driver.find_elements(By.CSS_SELECTOR, "img.rg_i")
        for image in images:
            try:
                # 이미지 src 속성 가져오기
                src = image.get_attribute("src")
                if src and src.startswith('http'):
                    image_urls.add(src)
                if len(image_urls) >= num_images:
                    break
            except Exception as e:
                print(f"Error while extracting image URL: {e}")
                pass

        # 페이지 스크롤 다운
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)
        new_height = driver.execute_script("return document.body.scrollHeight")
        if new_height == last_height:
            break
        last_height = new_height

    return list(image_urls)

def main():
    create_folder_if_not_exists(output_dir)
    
    for keyword in keywords:
        folder_path = os.path.join(output_dir, keyword.replace(" ", "_"))
        create_folder_if_not_exists(folder_path)
        
        image_urls = crawl_images(keyword)
        print(f"Found {len(image_urls)} images for keyword: {keyword}")
        for i, url in enumerate(image_urls):
            download_image(url, folder_path, i + 1)
    
    driver.quit()

if __name__ == "__main__":
    main()