Skip to content

Instantly share code, notes, and snippets.

@guruguruman
Created June 7, 2024 01:08
Show Gist options
  • Save guruguruman/172b36d2e76157bc2f1af1f671809742 to your computer and use it in GitHub Desktop.
Save guruguruman/172b36d2e76157bc2f1af1f671809742 to your computer and use it in GitHub Desktop.
Using Smartproxy API in TypeScript
export interface SerpScrapingResult {
query: string;
title: string;
position: number;
url: string;
}
function createRequestParams(apiKey: string, query: string): RequestInit {
return {
method: 'POST',
body: JSON.stringify({
target: 'google_search',
query: query,
parse: true,
domain: 'co.jp',
locale: 'ja',
google_results_language: 'ja',
geo: 'Japan',
device_type: 'mobile',
page_from: 1,
num_pages: 20,
google_nfpr: true,
google_safe_search: false,
}),
headers: {
'Content-Type': 'application/json',
Authorization: `Basic ${apiKey}`,
},
};
}
function createUrl(url: string): string {
const baseUrl = 'https://google.co.jp';
if (url.startsWith('http://') || url.startsWith('https://')) {
return url;
}
if (url.startsWith('/')) {
return `${baseUrl}${url}`;
} else {
return `${baseUrl}/${url}`;
}
}
function parseData(query: string, json: any): SerpScrapingResult[] {
const results = json.results[0].content.results;
const organics = results.organic;
return organics?.map((organic: any) => {
const pos = parseInt(organic.pos, 10);
return {
query,
title: organic.title || '',
position: isNaN(pos) ? null : pos,
url: createUrl(organic.url),
} as SerpScrapingResult;
}) || [];
}
export async function fetchSerp(apiKey: string, query: string): Promise<SerpScrapingResult[]> {
try {
const url = 'https://scrape.smartproxy.com/v1/tasks';
const params = createRequestParams(apiKey, query);
const response = await fetch(url, params).then((response) => {
if (!response.ok) {
throw new Error(`Error on fetch ${url}: ${response.statusText}`);
}
return response as Response;
});
const json = await response.json();
return parseData(query, json);
} catch (error) {
throw error;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment