query.csv

results.csv

webscrapping_inxight.py

import time

import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.by import By

chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--headless")

# linux 환경에서 필요한 option
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-dev-shm-usage")

# 크롬드라이버 실행
driver = webdriver.Chrome(options=chrome_options)

id = "3YK0326U7X"
url = "https://drugs.ncats.io/drug/" + id

# 크롬 드라이버에 url 주소 넣고 실행
driver.get(url)

# 페이지가 완전히 로딩되도록 2초동안 기다림
time.sleep(2)

# 토글 버튼을 찾아 변수에 저장 (By.XPATH 방식)
# toggle = driver.find_element(By.XPATH, "//*[@id="numbers"]")
# toggle.click()
# time.sleep(1)

driver.find_element(By.XPATH, '//*[@id="numbers"]/span/span[2]/i').click()
time.sleep(1)

hc_sequence = driver.find_element(
    By.XPATH, '//*[@id="subunits"]/div/subunit[1]/div/div[2]/div[1]/span/pre'
)
lc_sequence = driver.find_element(
    By.XPATH, '//*[@id="subunits"]/div/subunit[3]/div/div[2]/div[1]/span/pre'
)
# print(hc_sequence.text)


def scrapper(url_root, id):
    # 크롬 드라이버에 url 주소 넣고 실행
    # https://drugs.ncats.io/drug/
    driver.get(url_root + id)
    # 페이지가 완전히 로딩되도록 2초동안 기다림
    time.sleep(2)
    driver.find_element(By.XPATH, '//*[@id="numbers"]/span/span[2]/i').click()
    time.sleep(0.5)
    hc_sequence = driver.find_element(
        By.XPATH, '//*[@id="subunits"]/div/subunit[1]/div/div[2]/div[1]/span/pre'
    )
    lc_sequence = driver.find_element(
        By.XPATH, '//*[@id="subunits"]/div/subunit[3]/div/div[2]/div[1]/span/pre'
    )
    return hc_sequence.text, lc_sequence.text


def custom_function(row):
    try:
        # 예외가 발생할 수 있는 작업 수행
        driver.get("https://drugs.ncats.io/drug/" + row["entry_code"])
        # 페이지가 완전히 로딩되도록 2초동안 기다림
        time.sleep(2)
        driver.find_element(By.XPATH, '//*[@id="numbers"]/span/span[2]/i').click()
        time.sleep(0.5)
        try:
            hc_sequence = driver.find_element(
                By.XPATH,
                '//*[@id="subunits"]/div/subunit[1]/div/div[2]/div[1]/span/pre',
            )
            lc_sequence = driver.find_element(
                By.XPATH,
                '//*[@id="subunits"]/div/subunit[3]/div/div[2]/div[1]/span/pre',
            )
            return hc_sequence.text, lc_sequence.text  # 두 개의 결과를 튜플 형태로 반환
        except ValueError:
            return "No exist", "No exist"
    except ZeroDivisionError:
        # 예외 처리: 0으로 나누는 경우
        return "Divide by zero error"
    except KeyError:
        # 예외 처리: 특정 열이 존재하지 않는 경우
        return "Not exist"


if __name__ == "__main__":
    df = pd.read_csv("query.csv")
    df = df[~df["entry_code"].isin(["No entry", "No page"])]

    for i in df["entry_code"]:
        print(i)
        try:
            hc, lc = scrapper("https://drugs.ncats.io/drug/", i)
            df.loc[df["entry_code"] == i, "HeavyChain"] = hc
            df.loc[df["entry_code"] == i, "LightChain"] = lc
        except BaseException:
            hc, lc = "Not exist", "Not exist"
            df.loc[df["entry_code"] == i, "HeavyChain"] = hc
            df.loc[df["entry_code"] == i, "LightChain"] = lc

    df.to_csv("results.csv", index=False)