From 8d5ca942a6ed06d591b743f9d0ae6a551c51d0ee Mon Sep 17 00:00:00 2001 From: dragongoose Date: Sun, 26 Nov 2023 15:00:26 -0500 Subject: [PATCH] Working bulk stream info function, needs cleaning --- extractor/twitch/Streamer.go | 58 +++++++++++++++++++++++++++++++++--- extractor/twitch/parser.go | 20 +++++++++---- extractor/utils.go | 5 ++++ routes/api/users/users.go | 27 +++++++++-------- 4 files changed, 88 insertions(+), 22 deletions(-) diff --git a/extractor/twitch/Streamer.go b/extractor/twitch/Streamer.go index 4f9dcd2..8804582 100644 --- a/extractor/twitch/Streamer.go +++ b/extractor/twitch/Streamer.go @@ -2,6 +2,7 @@ package twitch import ( "errors" + "safetwitch-backend/extractor" "safetwitch-backend/extractor/structs" "github.com/tidwall/gjson" @@ -112,7 +113,7 @@ func GetStreamerInfo(streamerName string) (structs.Streamer, error) { return structs.Streamer{}, err } - parsedStream, err := ParseStream(string(body)) + parsedStream, err := ParseStream(string(body), 0) var isLive bool if err != nil { if err.Error() == "streamer is not live" { @@ -168,7 +169,7 @@ func GetStreamerId(channelName string) (string, error) { } } -func GetBulkStreamerInfo(streamers []string) string { +func GetBulkStreamerInfo(streamers []string) ([]structs.Streamer, error) { var payload []TwitchPayload for _, streamer := range streamers { tmp := []TwitchPayload{ @@ -261,7 +262,56 @@ func GetBulkStreamerInfo(streamers []string) string { payload = append(payload, tmp...) } - _, body, _ := parseResponse(payload) + _, body, err := parseResponse(payload) + if err != nil { + return []structs.Streamer{}, err + } - return string(body) + len := len(streamers) * 6 + + var foundStreamers []structs.Streamer + + for i := 0; i < (len / 6); i++ { + offset := i * 6 + + // begin parsing response + streamerFound := gjson.Get(string(body), extractor.GenGjsonQuery(offset, ".data.user")) + if streamerFound.String() == "" { + // return structs.Streamer{}, errors.New("streamer not found") + continue + } + + streamerData := gjson.Get(string(body), extractor.GenGjsonQuery(offset, ".data")) + parsedSocials, err := ParseSocials(streamerData.String()) + if err != nil { + // return structs.Streamer{}, err + continue + } + + parsedStream, err := ParseStream(string(body), offset) + var isLive bool + if err != nil { + if err.Error() == "streamer is not live" { + parsedStream = nil + } else { + continue + } + } + if parsedStream != nil { + isLive = true + } else { + isLive = false + } + + streamerBanner := gjson.Get(string(body), extractor.GenGjsonQuery(offset+5, ".data.userOrError.bannerImageURL")).String() + parsedStreamer, err := ParseStreamer(streamerData, isLive, parsedSocials, parsedStream, streamers[i], streamerBanner) + if err != nil { + return []structs.Streamer{}, err + } + + foundStreamers = append(foundStreamers, parsedStreamer) + + } + + return foundStreamers, nil } diff --git a/extractor/twitch/parser.go b/extractor/twitch/parser.go index b14315c..440f52a 100644 --- a/extractor/twitch/parser.go +++ b/extractor/twitch/parser.go @@ -66,15 +66,20 @@ func ParseSocials(data string) ([]structs.Social, error) { return parsedSocials, nil } -func ParseStream(data string) (*structs.Stream, error) { +// Takes in all of the data returned from twitch (see payload from Streamer.go) +// and parses it into a stream. Offset is used for bulk processing +// when the offsets are different. Use offset 0 for normal uses +func ParseStream(data string, offset int) (*structs.Stream, error) { // check if live - stream := gjson.Get(data, "1.data.user.stream") + // default "1.data.user.stream" + stream := gjson.Get(data, extractor.GenGjsonQuery(offset+1, ".data.user.stream")) if !stream.IsObject() { return nil, errors.New("streamer is not live") } var tags []string - tagArea := gjson.Get(data, "2.data.user.stream.freeformTags").Array() + // default "2.data.user.stream.freeformTags" + tagArea := gjson.Get(data, extractor.GenGjsonQuery(offset+2, ".data.user.stream.freeformTags")).Array() for _, tag := range tagArea { tags = append(tags, tag.Get("name").String()) } @@ -85,12 +90,15 @@ func ParseStream(data string) (*structs.Stream, error) { } parsedStream := structs.Stream{ - Title: gjson.Get(data, "1.data.user.lastBroadcast.title").String(), + // default "1.data.user.lastBroadcast.title" + Title: gjson.Get(data, extractor.GenGjsonQuery(offset+1, ".data.user.lastBroadcast.title")).String(), Topic: stream.Get("game.name").String(), StartedAt: time, Tags: tags, - Viewers: int(gjson.Get(data, "4.data.user.stream.viewersCount").Int()), - Preview: extractor.ProxyUrl(gjson.Get(data, "3.data.user.stream.previewImageURL").String()), + // default "4.data.user.stream.viewersCount" + Viewers: int(gjson.Get(data, extractor.GenGjsonQuery(offset+4, ".data.user.stream.viewersCount")).Int()), + // default "3.data.user.stream.previewImageURL" + Preview: extractor.ProxyUrl(gjson.Get(data, extractor.GenGjsonQuery(offset+3, ".data.user.stream.previewImageURL")).String()), } return &parsedStream, nil diff --git a/extractor/utils.go b/extractor/utils.go index 4459a5d..738b432 100644 --- a/extractor/utils.go +++ b/extractor/utils.go @@ -2,9 +2,14 @@ package extractor import ( b64 "encoding/base64" + "fmt" "os" ) +func GenGjsonQuery(i int, query string) string { + return fmt.Sprintf("%d%s", i, query) +} + func ProxyUrl(url string) string { encodedUrl := b64.URLEncoding.EncodeToString([]byte(url)) backendUrl := os.Getenv("URL") diff --git a/routes/api/users/users.go b/routes/api/users/users.go index 165f3d1..b042e61 100644 --- a/routes/api/users/users.go +++ b/routes/api/users/users.go @@ -28,18 +28,21 @@ func Routes(route *gin.Engine) { context.JSON(200, extractor.FormatMessage(data, true)) }) - // type postData struct { - // Streamers []string `json:"streamers"` - // } + type postData struct { + Streamers []string `json:"streamers"` + } - // auth.POST("/bulk", func(context *gin.Context) { - // var f postData - // err := context.ShouldBindJSON(&f) - // if err != nil { - // context.Error(err) - // } + auth.POST("/bulk", func(context *gin.Context) { + var f postData + err := context.ShouldBindJSON(&f) + if err != nil { + context.Error(err) + } - // data := twitch.GetBulkStreamerInfo(f.Streamers) - // context.JSON(200, extractor.FormatMessage(data, true)) - // }) + data, err := twitch.GetBulkStreamerInfo(f.Streamers) + if err != nil { + context.Error(err) + } + context.JSON(200, extractor.FormatMessage(data, true)) + }) }