bigkinds.or.kr 검색 크롤링 샘플 코드

crawl_bigkinds_or_kr.py

import json
import requests
from bs4 import BeautifulSoup

keyword_filters = {
    "filterProviderNm": "경향신문,세계일보",
    "filterCategoryNm": "",
    "filterIncidentCategoryNm": "",
    "filterProviderCode": "01100101,01100701",
    "filterCategoryCode": "",
    "filterIncidentCategoryCode": "",
    "filterDateCode": "",
    "filterAnalysisCode": "",
}

result_url = "https://www.bigkinds.or.kr/news/newsResult.do"

data = {
    'LOGIN_NAME': None,
    'LOGIN_SN': None,
    'byLine': '',
    'categoryCode': '',
    'categoryTab': '',
    'contentLength': '100',
    'dateCode': '',
    'endDate': '2017-07-04',
    'filterAnalysisCode': '',
    'filterCategoryCode': '',
    'filterDateCode': '',
    'filterIncidentCode': '',
    'filterProviderCode': '01100101,01100701',
    'highlighting': 'true',
    'incidentCode': '',
    'indexName': 'news',
    'interval': '',
    'keyword': '크롤링',
    'keywordFilterJson': json.dumps(keyword_filters),
    'keywordJson': '',
    'listMode': '',
    'login_chk': None,
    'mainTodayPersonYn': '',
    'newsId': '',
    'pageInfo': 'newsResult',
    'period': '3month',
    'providerCode': '',
    'quotationKeyword1': '',
    'quotationKeyword2': '',
    'quotationKeyword3': '',
    'realKeyword': '',
    'resultNumber': '10',
    'resultState': 'detailSearch',
    'searchFromUseYN': 'N',
    'searchFtr': '1',
    'searchScope': '1',
    'sessionUSID': '',
    'sessionUUID': 'test',
    'sortMethod': 'date',
    'startDate': '2017-04-04',
    'startNo': '1',
    'topmenuoff': '',
    'totalCount': '4',
}

response = requests.post(result_url, data=data)
response