Skip to content

Instantly share code, notes, and snippets.

@ariankordi
Last active September 10, 2018 22:58
Show Gist options
  • Save ariankordi/dd55358b5b3bbd759b6debe08dd1922b to your computer and use it in GitHub Desktop.
Save ariankordi/dd55358b5b3bbd759b6debe08dd1922b to your computer and use it in GitHub Desktop.
A Go script that extracts the "x minutes ago", "x hours ago", etc text from Miiverse clones like https://cedar.doctor/titles/1, then averages them out and sees the average seconds between each post. Might be broken if these sites are down or IPs are just dead.
package main
import (
// requests
"github.com/valyala/fasthttp"
// for spltting bytes
"bytes"
// for getting the things from the thing
"regexp"
// for errors
"errors"
"strconv"
// count time
"time"
// printing in main()
"fmt"
// error fatal
//"log"
)
// fasthttp vars stuff
var req = fasthttp.AcquireRequest()
var resp = fasthttp.AcquireResponse()
var client = &fasthttp.Client{}
// regex pattern
// this didn't work on ciiverse so we're making another one
//var pattern = regexp.MustCompile(`(?:"\/posts\/.*">)(.*ago+)(?:<)`)
// also didn't need the non capturing groups
var pattern = regexp.MustCompile(`"timestamp".*>(.*ago+)<`)
// lol []byte sorry
// Plain HTTP is prefered for, like. 0.0005% less overhead.
func GetApproxSecIntervalForPage(url string) (int, error) {
req.SetRequestURI(url)
err := client.Do(req, resp)
if err != nil {
return 0, err
}
if resp.StatusCode() != 200 {
return 0, errors.New("http response code " + strconv.Itoa(resp.StatusCode()))
}
body := resp.Body()
//fmt.Println(string(body))
matches := pattern.FindAllSubmatch(body, -1)
// empty int slice we will put our seconds into
intSlice := []int{}
for i, _ := range matches {
// matches[i][1] is our matched thing, "30 minutes ago" etc
// this would be "30 minutes"
noAgo := bytes.Split(matches[i][1], []byte(" ago"))[0]
// this is "minutes", EXCEPT
unitThing := bytes.Split(noAgo, []byte(" "))
// if it has more than 3 elements, then don't do this
//fmt.Println(len(unitThing))
if len(unitThing) < 3 {
//fmt.Println("doing it")
// THIS is "minutes"
unit := unitThing[1]
// and this is "30"
numStr := string(unitThing[0])
num, err := strconv.Atoi(numStr)
if err != nil {
unit = []byte{}
//fmt.Println(err)
}
// second, secondS
if num != 1 {
unit = unit[:len(unit)-1]
}
seconds := 0
//fmt.Println(string(unit))
// can't compare bytes
switch(string(unit)) {
case "second":
seconds = num
case "minute":
seconds = num * 60
case "hour":
seconds = num * 3600
case "day":
seconds = num * 86400
default:
seconds = 0
}
// append to int slice if the seconds is not 0, duh
if seconds != 0 {
intSlice = append(intSlice, seconds)
}
}
}
//fmt.Println(intSlice)
// average it all
total := 0
for i, _ := range intSlice {
total += intSlice[i]
}
// this prevents a division by 0 and a panic
if total == 0 || len(intSlice) == 0 {
return 0, errors.New("total is 0 or intSlice is 0")
}
avg := total / len(intSlice)
return avg, nil
}
// This is just a thing here idk this could be ignored if this were an actual package
func main() {
// set some headers for some sites
// get a smaller PJAX version for closedverse and cedar servers
req.Header.Add("X-PJAX", "1")
// add nintendo user agent for closed.pizza, to use plain http
req.Header.Add("User-Agent", "Nintendo")
fmt.Println("getting thing for closedpizza")
start := time.Now()
point, err := GetApproxSecIntervalForPage("http://closed.pizza/communities/6")
elapsed := time.Since(start)
if err != nil {
//log.Fatal("failed getting thing for closedpizza: ", err)
fmt.Println("failed getting thing for closedpizza:", err)
} else {
fmt.Println(point, "(" + elapsed.String() + ")")
}
fmt.Println("getting thing for oanus")
start = time.Now()
point, err = GetApproxSecIntervalForPage("http://206.189.235.47/communities/1")
elapsed = time.Since(start)
if err != nil {
//log.Fatal("failed getting thing for oanus: ", err)
fmt.Println("failed getting thing for oanus:", err)
} else {
fmt.Println(point, "(" + elapsed.String() + ")")
}
// add cf-connecting-ip header for cedar, the others don't like this
req.Header.Add("CF-Connecting-IP", "1")
fmt.Println("getting thing for cedardoctor")
start = time.Now()
point, err = GetApproxSecIntervalForPage("http://167.99.10.132/titles/1")
elapsed = time.Since(start)
if err != nil {
//log.Fatal("failed getting thing for cedardoctor: ", err)
fmt.Println("failed getting thing for cedardoctor:", err)
} else {
fmt.Println(point, "(" + elapsed.String() + ")")
}
fmt.Println("getting thing for ciiverse")
start = time.Now()
point, err = GetApproxSecIntervalForPage("http://ciiverse.ga/communities/5")
elapsed = time.Since(start)
if err != nil {
//log.Fatal("failed getting thing for ciiverse: ", err)
fmt.Println("failed getting thing for ciiverse:", err)
} else {
fmt.Println(point, "(" + elapsed.String() + ")")
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment