Skip to content

Instantly share code, notes, and snippets.

@InJeCTrL

InJeCTrL/get.py

Created April 6, 2021 03:56
Show Gist options
  • Save InJeCTrL/397082cebac22b99da5a795f2dee5c40 to your computer and use it in GitHub Desktop.
Save InJeCTrL/397082cebac22b99da5a795f2dee5c40 to your computer and use it in GitHub Desktop.
Download ZIP
凹凸租车爬虫+web
Raw
get.py
import requests
import time
import json
import pymysql
# 310100 上海
# 320100 南京
# 330100 杭州
# 440100 广州
# 440300 深圳
# 110100 北京
# 420100 武汉
# 510100 成都
# 370200 青岛
db = pymysql.connect("localhost", "root", "", "CarTest", charset='utf8')
cursor = db.cursor()
url = "https://www.atzuche.com/appserver/h5/v592/search/car/list"
header = {
'Host': 'www.atzuche.com',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0',
'Accept': 'application/json;version=3.0;compress=false',
'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Accept-Encoding': 'gzip, deflate, br',
'Referer': 'https://www.atzuche.com/qd/s/',
'Content-Type': 'application/json;charset=utf-8',
'Origin': 'https://www.atzuche.com',
'Connection': 'keep-alive',
'Cookie': 'xxx',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache'
}
data = {
'cityCode': "370200",
'filterCondition': {
'seq': 4
},
'pageNum': 1,
'pageSize': 20,
'requestId': int(time.time() * 1000),
'sceneCode': 'U005',
'searchType': 1
}
totalPage = 99999
for i in range(1, totalPage):
data["pageNum"] = i
with requests.post(url, headers = header, data = json.dumps(data)) as response:
txt = response.text
d = json.loads(txt)
carList = d["data"]["carList"]
for car in carList:
sql = "insert into car value(%d, \"%s\", \"%s\", \"%s\", \"%s\", %d, \"%s\", \"%s\", \"%s\", \"%s\", %f, %d, \"%s\", \"%s\")" % (int(car["carNo"]), "青岛", car["brandInfo"], car["sweptVolum"] if "sweptVolum" in car else "", "https://carphoto.atzuche.com/" + car["coverPic"], int(car["dayPrice"]), car["distance"], car["gbType"], car["plateNum"], car["dayMileageTxt"], float(car["carScore"]), int(car["brand"]), car["typeTxt"], car["carAddressBO"]["districtName"])
try:
cursor.execute(sql)
except Exception as e:
print(e)
print(sql)
db.close()
Raw
getBrands.py
import requests
import time
import json
import pymysql
db = pymysql.connect("localhost", "root", "", "CarTest", charset='utf8')
cursor = db.cursor()
url = "https://www.atzuche.com/appserver/h5/web/carParam/listBrandMap/370200?requestId=" + str(int(time.time() * 1000))
with requests.get(url) as response:
txt = response.text
d = json.loads(txt)
data = d["data"]["brandSortMap"]
for ch in data:
for brand in data[ch]:
sql = "insert into brand value(%d, \"%s\", \"%s\")" % (brand["id"], brand["txt"], ch)
print(sql)
cursor.execute(sql)
db.close()
Raw
index.html
<html>
<head>
<title>租车数据查看</title>
<script src="https://cdn.bootcdn.net/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
</head>
<body>
<div>
<div>筛选：</div>
<div>
<span>城市：</span>
<select id="city">
<option value="-1" selected="selected">
全部
</option>
</select>
</div>
<div>
<span>品牌：</span>
<select id="brand">
<option value="-1" selected="selected">
全部
</option>
</select>
</div>
<div>
<span>区域：</span>
<select id="district">
<option value="-1" selected="selected">
全部
</option>
</select>
</div>
<div>
<span>价格区间：</span>
<input id="startprice" type="number" value="0" />~
<input id="endprice" type="number" value="-1" />(最大填-1代表不限制最大值)
</div>
<div>
<button id="search">搜索</button>
</div>
</div>
<table id="list" style="margin: auto;" border="1">
<tr>
<th>品牌</th>
<th>型号</th>
<th>照片</th>
<th>日限制</th>
<th>车辆评分</th>
<th>日租价格</th>
<th>里程数</th>
<th>类型</th>
<th>区域</th>
<th>城市</th>
<th>排量</th>
<th>部分牌照</th>
<th>跳转到车辆主页</th>
</tr>
</table>
<div style="text-align: center;">
<span>检索到：</span>
<span id="count">0</span>
<span>条</span>
</div>
<div style="text-align: center;">
<button id="prev"><</button>
<button id="next">></button>
</div>
</body>
<script>
var city = "-1";
var brandId = "-1";
var district = "-1";
var startprice = "0";
var endprice = "-1";
var startidx = 0;
function getCarList(){
city = $("#city").val();
brandId = $("#brand").val();
district = $("#district").val();
startprice = $("#startprice").val();
endprice = $("#endprice").val();
$.ajax({
type: "GET",
url: "./car",
data: {
city: city,
brand: brandId,
district: district,
startprice: startprice,
endprice: endprice,
startidx: startidx
},
success: function(d){
$("#count").text(d["count"]);
$("#list").html("<tr><th>品牌</th><th>型号</th><th>照片</th><th>日限制</th><th>车辆评分</th><th>日租价格</th><th>里程数</th>" +
"<th>类型</th><th>区域</th><th>城市</th><th>排量</th><th>部分牌照</th><th>跳转到车辆主页</th></tr>");
for (let i = 0; i < d["carList"].length; ++i){
$("#list").append(
"<tr><td>" + d["carList"][i]["brand"] + "</td>" +
"<td>" + d["carList"][i]["typeTxt"] + "</td>" +
"<td><img src=\"" + d["carList"][i]["coverPic"] + "\"></img></td>" +
"<td>" + d["carList"][i]["dayMileageTxt"] + "</td>" +
"<td>" + d["carList"][i]["carScore"] + "</td>" +
"<td>" + d["carList"][i]["dayPrice"] + "</td>" +
"<td>" + d["carList"][i]["distance"] + "</td>" +
"<td>" + d["carList"][i]["gbType"] + "</td>" +
"<td>" + d["carList"][i]["districtName"] + "</td>" +
"<td>" + d["carList"][i]["city"] + "</td>" +
"<td>" + d["carList"][i]["sweptVolum"] + "</td>" +
"<td>" + d["carList"][i]["plateNum"] + "</td>" +
"<td><a target=_blank href=\"https://www.atzuche.com/cardetail/" + d["carList"][i]["carNo"] + "\">跳转</a></td></tr>"
);
}
}
});
}
$("#search").bind('click', function(e){
getCarList();
});
$("#prev").bind('click', function(e){
if (startidx >= 20){
startidx -= 20;
getCarList();
}
});
$("#next").bind('click', function(e){
startidx += 20;
getCarList();
});
$.ajax({
type: "GET",
url: "./brand",
success: function(d){
for (let i = 0; i < d["brandList"].length; ++i){
$("#brand").append($("<option>").val(d["brandList"][i]["brandId"]).text(d["brandList"][i]["Name"]));
}
}
});
$.ajax({
type: "GET",
url: "./city",
success: function(d){
for (let i = 0; i < d["cityList"].length; ++i){
$("#city").append($("<option>").val(d["cityList"][i]).text(d["cityList"][i]));
}
}
});
$.ajax({
type: "GET",
url: "./district",
success: function(d){
for (let i = 0; i < d["districtList"].length; ++i){
$("#district").append($("<option>").val(d["districtList"][i]).text(d["districtList"][i]));
}
}
});
</script>
</html>
Raw
server.py
from flask import Flask, jsonify, send_file, request, render_template
import pymysql
app = Flask(__name__)
@app.route("/car", methods = ["GET"])
def getCarList():
in_city = request.values.get("city")
in_brand = request.values.get("brand")
in_district = request.values.get("district")
in_startprice = request.values.get("startprice")
in_endprice = request.values.get("endprice")
in_startidx = request.values.get("startidx")
db = pymysql.connect("localhost", "root", "", "CarTest", charset='utf8')
cursor = db.cursor()
sql = "select count(*) from car where"
if in_brand != "-1":
sql += " brandId = " + in_brand + " and"
if in_city != "-1":
sql += " city = " + in_city + " and"
if in_district != "-1":
sql += " districtName = " + in_district + " and"
sql += " dayPrice >= " + str(in_startprice) + " and"
if in_endprice != "-1":
sql += " dayPrice <= " + str(in_endprice)
if sql.endswith("and"):
sql = sql[:sql.rindex(" and")]
if sql.endswith("where"):
sql = sql[:sql.rindex("where")]
cursor.execute(sql)
data = cursor.fetchone()
ret = {}
ret["count"] = data[0]
ret["carList"] = []
sql = "select carNo, Name, typeTxt, sweptVolum, coverPic, dayPrice, distance, gbType, plateNum, dayMileageTxt, carScore, districtName, city from car, brand where car.brandId = brand.Id and"
if in_brand != "-1":
sql += " brandId = " + in_brand + " and"
if in_city != "-1":
sql += " city = " + in_city + " and"
if in_district != "-1":
sql += " districtName = " + in_district + " and"
sql += " dayPrice >= " + str(in_startprice) + " and"
if in_endprice != "-1":
sql += " dayPrice <= " + str(in_endprice)
if sql.endswith("and"):
sql = sql[:sql.rindex(" and")]
if sql.endswith("where "):
sql = sql[:sql.rindex("where ")]
sql += " limit " + in_startidx + ",20"
# print(sql)
cursor.execute(sql)
results = cursor.fetchall()
db.close()
for result in results:
car = {}
car["carNo"] = result[0]
car["brand"] = result[1]
car["typeTxt"] = result[2]
car["sweptVolum"] = result[3]
car["coverPic"] = result[4]
car["dayPrice"] = result[5]
car["distance"] = result[6]
car["gbType"] = "自动挡" if result[7] == "2" else ("手动挡" if result[7] == "1" else "")
car["plateNum"] = result[8]
car["dayMileageTxt"] = result[9]
car["carScore"] = result[10]
car["districtName"] = result[11]
car["city"] = result[12]
ret["carList"].append(car)
return ret
@app.route("/city", methods = ["GET"])
def getCityList():
db = pymysql.connect("localhost", "root", "", "CarTest", charset='utf8')
cursor = db.cursor()
sql = "select distinct city from car"
cursor.execute(sql)
results = cursor.fetchall()
db.close()
ret = {}
ret["cityList"] = []
for result in results:
ret["cityList"].append(result[0])
return ret
@app.route("/district", methods = ["GET"])
def getDistrictList():
db = pymysql.connect("localhost", "root", "", "CarTest", charset='utf8')
cursor = db.cursor()
sql = "select distinct districtName from car"
cursor.execute(sql)
results = cursor.fetchall()
db.close()
ret = {}
ret["districtList"] = []
for result in results:
ret["districtList"].append(result[0])
return ret
@app.route("/brand", methods = ["GET"])
def getBrandList():
db = pymysql.connect("localhost", "root", "", "CarTest", charset='utf8')
cursor = db.cursor()
sql = "select * from brand"
cursor.execute(sql)
results = cursor.fetchall()
db.close()
ret = {}
ret["brandList"] = []
for result in results:
ret["brandList"].append({"brandId": result[0], "Name": result[1]})
return ret
@app.route("/", methods = ["GET"])
def mapindex():
return send_file('./index.html')
app.run(host='127.0.0.1', port = 5555)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment