diff --git a/extractor/twitch/Streamer.go b/extractor/twitch/Streamer.go index 8804582..30b6097 100644 --- a/extractor/twitch/Streamer.go +++ b/extractor/twitch/Streamer.go @@ -2,8 +2,10 @@ package twitch import ( "errors" + "fmt" "safetwitch-backend/extractor" "safetwitch-backend/extractor/structs" + "sync" "github.com/tidwall/gjson" ) @@ -169,149 +171,78 @@ func GetStreamerId(channelName string) (string, error) { } } -func GetBulkStreamerInfo(streamers []string) ([]structs.Streamer, error) { - var payload []TwitchPayload +func BulkCheckIfStreamerIsLive(streamers []string) ([]string, error) { + // We can only make 35 different queries in the same request + // or else it throws 400 error :( So here, payload will be + // an array of an array, which we will make requests in + // 35 segments + var payloads [][]TwitchPayload + var tmparr []TwitchPayload for _, streamer := range streamers { - tmp := []TwitchPayload{ - // Streamer data - { - "operationName": "ChannelRoot_AboutPanel", - "variables": map[string]interface{}{ - "channelLogin": streamer, - "skipSchedule": false, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6", - }, - }, - }, + if len(tmparr) > 34 { + // add full tmparr to payloads + payloads = append(payloads, tmparr) + // empty tmp arr + tmparr = []TwitchPayload{} + } - // Stream metadata - { - "operationName": "StreamMetadata", - "variables": map[string]interface{}{ - "channelLogin": streamer, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962", - }, - }, + tmp := TwitchPayload{ + "operationName": "UseLive", + "variables": map[string]interface{}{ + "channelLogin": streamer, }, - - // Stream tags - { - "operationName": "StreamTagsTrackingChannel", - "variables": map[string]interface{}{ - "channel": streamer, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "6aa3851aaaf88c320d514eb173563d430b28ed70fdaaf7eeef6ed4b812f48608", - }, - }, - }, - - // Stream preview image - { - "operationName": "VideoPreviewOverlay", - "variables": map[string]interface{}{ - "login": streamer, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f", - }, - }, - }, - - // Current views - { - "operationName": "UseViewCount", - "variables": map[string]interface{}{ - "channelLogin": streamer, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "00b11c9c428f79ae228f30080a06ffd8226a1f068d6f52fbc057cbde66e994c2", - }, - }, - }, - - // Get offline banner image - { - "operationName": "ChannelShell", - "variables": map[string]interface{}{ - "login": streamer, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "580ab410bcd0c1ad194224957ae2241e5d252b2c5173d8e0cce9d32d5bb14efe", - }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "639d5f11bfb8bf3053b424d9ef650d04c4ebb7d94711d644afb08fe9a0fad5d9", }, }, } - payload = append(payload, tmp...) + tmparr = append(tmparr, tmp) + } + if len(tmparr) > 0 { + payloads = append(payloads, tmparr) } - _, body, err := parseResponse(payload) - if err != nil { - return []structs.Streamer{}, err + bodies := []string{} + bodiesMutex := sync.Mutex{} + wg := sync.WaitGroup{} + ch := make(chan error, len(payloads)) + + for _, payload := range payloads { + wg.Add(1) + + go func(payload []TwitchPayload) { + defer wg.Done() + + _, body, err := parseResponse(payload) + if err != nil { + ch <- err + return + } + + bodiesMutex.Lock() + bodies = append(bodies, string(body)) + bodiesMutex.Unlock() + }(payload) } + wg.Wait() + fmt.Println(bodies) - len := len(streamers) * 6 + var liveStreamers []string - var foundStreamers []structs.Streamer + // parse the data from all of the segments before + for _, body := range bodies { + fmt.Println(body) + for i := 0; i < len(streamers); i++ { + stream := gjson.Get(string(body), extractor.GenGjsonQuery(i, ".data.user.stream")) - for i := 0; i < (len / 6); i++ { - offset := i * 6 - - // begin parsing response - streamerFound := gjson.Get(string(body), extractor.GenGjsonQuery(offset, ".data.user")) - if streamerFound.String() == "" { - // return structs.Streamer{}, errors.New("streamer not found") - continue - } - - streamerData := gjson.Get(string(body), extractor.GenGjsonQuery(offset, ".data")) - parsedSocials, err := ParseSocials(streamerData.String()) - if err != nil { - // return structs.Streamer{}, err - continue - } - - parsedStream, err := ParseStream(string(body), offset) - var isLive bool - if err != nil { - if err.Error() == "streamer is not live" { - parsedStream = nil - } else { - continue + if stream.IsObject() { + liveStreamers = append(liveStreamers, streamers[i]) } } - if parsedStream != nil { - isLive = true - } else { - isLive = false - } - - streamerBanner := gjson.Get(string(body), extractor.GenGjsonQuery(offset+5, ".data.userOrError.bannerImageURL")).String() - parsedStreamer, err := ParseStreamer(streamerData, isLive, parsedSocials, parsedStream, streamers[i], streamerBanner) - if err != nil { - return []structs.Streamer{}, err - } - - foundStreamers = append(foundStreamers, parsedStreamer) - } - return foundStreamers, nil + return liveStreamers, nil } diff --git a/routes/api/users/users.go b/routes/api/users/users.go index d8bd750..72c7636 100644 --- a/routes/api/users/users.go +++ b/routes/api/users/users.go @@ -32,7 +32,7 @@ func Routes(route *gin.Engine) { Streamers []string `json:"streamers"` } - auth.POST("/bulk", func(context *gin.Context) { + auth.POST("/isLive/bulk", func(context *gin.Context) { var f postData err := context.ShouldBindJSON(&f) if err != nil { @@ -40,7 +40,7 @@ func Routes(route *gin.Engine) { return } - data, err := twitch.GetBulkStreamerInfo(f.Streamers) + data, err := twitch.BulkCheckIfStreamerIsLive(f.Streamers) if err != nil { context.Error(err) return