Created
March 14, 2024 11:46
-
-
Save suzuki-shunsuke/042f15cb1e3fa7d2b30366a74c09a2a2 to your computer and use it in GitHub Desktop.
A HTTP Request to crates.io with Go causes 403 (violation of crates.io crawler policy)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"fmt" | |
"io" | |
"log" | |
"net/http" | |
"os" | |
) | |
func main() { | |
if err := core(); err != nil { | |
log.Fatal(err) | |
} | |
} | |
func core() error { | |
// curl -o /dev/null -w '%{http_code}\n' -s https://crates.io/api/v1/crates/skim/versions | |
// 200 | |
req, err := http.NewRequest("GET", "https://crates.io/api/v1/crates/skim/versions", nil) | |
if err != nil { | |
return err | |
} | |
resp, err := http.DefaultClient.Do(req) | |
if err != nil { | |
return err | |
} | |
defer resp.Body.Close() | |
fmt.Println(resp.StatusCode) // 403 | |
// {"errors":[{"detail":"We are unable to process your request at this time. This usually means that you are in violation of our crawler policy (https://crates.io/policies#crawlers). Please open an issue at https://github.com/rust-lang/crates.io or email [email protected] and provide the request id xxx"}]} | |
if _, err := io.Copy(os.Stdout, resp.Body); err != nil { | |
return err | |
} | |
return nil | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment