Bulk "is live" endpoint

extractor/twitch/Streamer.go

@@ -2,8 +2,10 @@ package twitch
 
 import (
 	"errors"
+	"fmt"
 	"safetwitch-backend/extractor"
 	"safetwitch-backend/extractor/structs"
+	"sync"
 
 	"github.com/tidwall/gjson"
 )
@@ -169,149 +171,78 @@ func GetStreamerId(channelName string) (string, error) {
 	}
 }
 
-func GetBulkStreamerInfo(streamers []string) ([]structs.Streamer, error) {
+func BulkCheckIfStreamerIsLive(streamers []string) ([]string, error) {
-	var payload []TwitchPayload
+	// We can only make 35 different queries in the same request
+	// or else it throws 400 error :( So here, payload will be
+	// an array of an array, which we will make requests in
+	// 35 segments
+	var payloads [][]TwitchPayload
+	var tmparr []TwitchPayload
 	for _, streamer := range streamers {
-		tmp := []TwitchPayload{
+		if len(tmparr) > 34 {
-			// Streamer data
+			// add full tmparr to payloads
-			{
+			payloads = append(payloads, tmparr)
-				"operationName": "ChannelRoot_AboutPanel",
+			// empty tmp arr
-				"variables": map[string]interface{}{
+			tmparr = []TwitchPayload{}
-					"channelLogin": streamer,
+		}
-					"skipSchedule": false,
-				},
-				"extensions": map[string]interface{}{
-					"persistedQuery": map[string]interface{}{
-						"version":    1,
-						"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
-					},
-				},
-			},
 
-			// Stream metadata
+		tmp := TwitchPayload{
-			{
+			"operationName": "UseLive",
-				"operationName": "StreamMetadata",
+			"variables": map[string]interface{}{
-				"variables": map[string]interface{}{
+				"channelLogin": streamer,
-					"channelLogin": streamer,
-				},
-				"extensions": map[string]interface{}{
-					"persistedQuery": map[string]interface{}{
-						"version":    1,
-						"sha256Hash": "a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962",
-					},
-				},
 			},
-
+			"extensions": map[string]interface{}{
-			// Stream tags
+				"persistedQuery": map[string]interface{}{
-			{
+					"version":    1,
-				"operationName": "StreamTagsTrackingChannel",
+					"sha256Hash": "639d5f11bfb8bf3053b424d9ef650d04c4ebb7d94711d644afb08fe9a0fad5d9",
-				"variables": map[string]interface{}{
-					"channel": streamer,
-				},
-				"extensions": map[string]interface{}{
-					"persistedQuery": map[string]interface{}{
-						"version":    1,
-						"sha256Hash": "6aa3851aaaf88c320d514eb173563d430b28ed70fdaaf7eeef6ed4b812f48608",
-					},
-				},
-			},
-
-			// Stream preview image
-			{
-				"operationName": "VideoPreviewOverlay",
-				"variables": map[string]interface{}{
-					"login": streamer,
-				},
-				"extensions": map[string]interface{}{
-					"persistedQuery": map[string]interface{}{
-						"version":    1,
-						"sha256Hash": "9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f",
-					},
-				},
-			},
-
-			// Current views
-			{
-				"operationName": "UseViewCount",
-				"variables": map[string]interface{}{
-					"channelLogin": streamer,
-				},
-				"extensions": map[string]interface{}{
-					"persistedQuery": map[string]interface{}{
-						"version":    1,
-						"sha256Hash": "00b11c9c428f79ae228f30080a06ffd8226a1f068d6f52fbc057cbde66e994c2",
-					},
-				},
-			},
-
-			// Get offline banner image
-			{
-				"operationName": "ChannelShell",
-				"variables": map[string]interface{}{
-					"login": streamer,
-				},
-				"extensions": map[string]interface{}{
-					"persistedQuery": map[string]interface{}{
-						"version":    1,
-						"sha256Hash": "580ab410bcd0c1ad194224957ae2241e5d252b2c5173d8e0cce9d32d5bb14efe",
-					},
 				},
 			},
 		}
 
-		payload = append(payload, tmp...)
+		tmparr = append(tmparr, tmp)
+	}
+	if len(tmparr) > 0 {
+		payloads = append(payloads, tmparr)
 	}
 
-	_, body, err := parseResponse(payload)
+	bodies := []string{}
-	if err != nil {
+	bodiesMutex := sync.Mutex{}
-		return []structs.Streamer{}, err
+	wg := sync.WaitGroup{}
+	ch := make(chan error, len(payloads))
+
+	for _, payload := range payloads {
+		wg.Add(1)
+
+		go func(payload []TwitchPayload) {
+			defer wg.Done()
+
+			_, body, err := parseResponse(payload)
+			if err != nil {
+				ch <- err
+				return
+			}
+
+			bodiesMutex.Lock()
+			bodies = append(bodies, string(body))
+			bodiesMutex.Unlock()
+		}(payload)
 	}
+	wg.Wait()
+	fmt.Println(bodies)
 
-	len := len(streamers) * 6
+	var liveStreamers []string
 
-	var foundStreamers []structs.Streamer
+	// parse the data from all of the segments before
+	for _, body := range bodies {
+		fmt.Println(body)
+		for i := 0; i < len(streamers); i++ {
+			stream := gjson.Get(string(body), extractor.GenGjsonQuery(i, ".data.user.stream"))
 
-	for i := 0; i < (len / 6); i++ {
+			if stream.IsObject() {
-		offset := i * 6
+				liveStreamers = append(liveStreamers, streamers[i])
-
-		// begin parsing response
-		streamerFound := gjson.Get(string(body), extractor.GenGjsonQuery(offset, ".data.user"))
-		if streamerFound.String() == "" {
-			// return structs.Streamer{}, errors.New("streamer not found")
-			continue
-		}
-
-		streamerData := gjson.Get(string(body), extractor.GenGjsonQuery(offset, ".data"))
-		parsedSocials, err := ParseSocials(streamerData.String())
-		if err != nil {
-			// return structs.Streamer{}, err
-			continue
-		}
-
-		parsedStream, err := ParseStream(string(body), offset)
-		var isLive bool
-		if err != nil {
-			if err.Error() == "streamer is not live" {
-				parsedStream = nil
-			} else {
-				continue
 			}
 		}
-		if parsedStream != nil {
-			isLive = true
-		} else {
-			isLive = false
-		}
-
-		streamerBanner := gjson.Get(string(body), extractor.GenGjsonQuery(offset+5, ".data.userOrError.bannerImageURL")).String()
-		parsedStreamer, err := ParseStreamer(streamerData, isLive, parsedSocials, parsedStream, streamers[i], streamerBanner)
-		if err != nil {
-			return []structs.Streamer{}, err
-		}
-
-		foundStreamers = append(foundStreamers, parsedStreamer)
-
 	}
 
-	return foundStreamers, nil
+	return liveStreamers, nil
 }

routes/api/users/users.go

@@ -32,7 +32,7 @@ func Routes(route *gin.Engine) {
 		Streamers []string `json:"streamers"`
 	}
 
-	auth.POST("/bulk", func(context *gin.Context) {
+	auth.POST("/isLive/bulk", func(context *gin.Context) {
 		var f postData
 		err := context.ShouldBindJSON(&f)
 		if err != nil {
@@ -40,7 +40,7 @@ func Routes(route *gin.Engine) {
 			return
 		}
 
-		data, err := twitch.GetBulkStreamerInfo(f.Streamers)
+		data, err := twitch.BulkCheckIfStreamerIsLive(f.Streamers)
 		if err != nil {
 			context.Error(err)
 			return