Split fetchers into multiple files for readability

extractor/twitch/Badges.go

@@ -0,0 +1,66 @@
+package twitch
+
+import (
+	"safetwitch-backend/extractor/structs"
+
+	"github.com/tidwall/gjson"
+)
+
+func GetTwitchBadges() ([]structs.Badge, error) {
+	payload := []TwitchPayload{
+		{
+			"operationName": "ChannelPointsPredictionBadges",
+			"variables":     map[string]interface{}{},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"sha256Hash": "36995b30b22c31d1cd0aa329987ac9b5368bb7e6e1ab1df42808bdaa80a6dbf9",
+					"version":    1,
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return []structs.Badge{}, err
+	}
+
+	rawBadges := gjson.Get(string(body), "0.data.badges")
+	formattedBadges, err := ParseBadges(rawBadges)
+	if err != nil {
+		return []structs.Badge{}, err
+	}
+
+	return formattedBadges, nil
+}
+
+func GetStreamerBadges(streamerName string) ([]structs.Badge, error) {
+	payload := []TwitchPayload{
+		{
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"sha256Hash": "86f43113c04606e6476e39dcd432dee47c994d77a83e54b732e11d4935f0cd08",
+					"version":    1,
+				},
+			},
+			"operationName": "ChatList_Badges",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return []structs.Badge{}, err
+	}
+
+	rawBadges := gjson.Get(string(body), "0.data.user.broadcastBadges")
+
+	formattedBadges, err := ParseBadges(rawBadges)
+	if err != nil {
+		return []structs.Badge{}, err
+	}
+
+	return formattedBadges, nil
+}

extractor/twitch/Discover.go

@@ -0,0 +1,130 @@
+package twitch
+
+import (
+	"safetwitch-backend/extractor"
+	"safetwitch-backend/extractor/structs"
+
+	"github.com/tidwall/gjson"
+)
+
+func GetDiscoveryPage(limit int, cursor string) ([]structs.CategoryPreview, error) {
+	payload := []TwitchPayload{
+		{
+			"operationName": "BrowsePage_AllDirectories",
+			"variables": map[string]interface{}{
+				"limit": limit,
+				"options": map[string]interface{}{
+					"recommendationsContext": map[string]interface{}{
+						"platform": "web",
+					},
+					"requestID": "JIRA-VXP-2397",
+					"sort":      "RELEVANCE",
+					"tags":      []string{},
+				},
+				"cursor": cursor,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "1d1914ca3cbfaa607ecd5595b2e305e96acf987c8f25328f7713b25f604c4668",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return []structs.CategoryPreview{}, err
+	}
+
+	var parsedCategoryArray []structs.CategoryPreview
+	categoryArray := gjson.Get(string(body), "0.data.directoriesWithTags.edges")
+	for _, categoryRes := range categoryArray.Array() {
+		parsedCategory, err := ParseCategory(categoryRes)
+		if err != nil {
+			return []structs.CategoryPreview{}, nil
+		}
+
+		parsedCategoryArray = append(parsedCategoryArray, parsedCategory)
+	}
+
+	return parsedCategoryArray, nil
+}
+
+func GetDiscoveryItem(name string, streamLimit int, cursor string) (structs.CategoryData, error) {
+	payload := []TwitchPayload{
+		{
+			"operationName": "DirectoryPage_Game",
+			"variables": map[string]interface{}{
+				"cursor":     cursor,
+				"imageWidth": 50,
+				"name":       name,
+				"options": map[string]interface{}{
+					"sort": "RELEVANCE",
+					"recommendationsContext": map[string]interface{}{
+						"platform": "web",
+					},
+					"requestID":    "JIRA-VXP-2397",
+					"freeformTags": nil,
+					"tags":         []string{},
+				},
+				"sortTypeIsRecency": false,
+				"limit":             streamLimit,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "df4bb6cc45055237bfaf3ead608bbafb79815c7100b6ee126719fac3762ddf8b",
+				},
+			},
+		},
+		{
+			"operationName": "Directory_DirectoryBanner",
+			"variables": map[string]interface{}{
+				"name": name,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "2670fbecd8fbea0211c56528d6eff5752ef9d6c73cd5238d395784b46335ded4",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return structs.CategoryData{}, err
+	}
+
+	categoryStreams := gjson.Get(string(body), "0.data.game.streams.edges")
+
+	var parsedStreams []structs.CategoryMinifiedStream
+	for _, stream := range categoryStreams.Array() {
+		parsed, err := ParseMinifiedStream(stream)
+		if err != nil {
+			return structs.CategoryData{}, err
+		}
+		parsedStreams = append(parsedStreams, parsed)
+	}
+
+	categoryData := gjson.Get(string(body), "1.data.game")
+
+	var tags []string
+	for _, tag := range categoryData.Get("tags").Array() {
+		tags = append(tags, tag.Get("localizedName").String())
+	}
+
+	parsedCategory := structs.CategoryData{
+		Name:        categoryData.Get("name").String(),
+		DisplayName: categoryData.Get("displayName").String(),
+		Description: categoryData.Get("description").String(),
+		Viewers:     int(categoryData.Get("viewersCount").Int()),
+		Followers:   int(categoryData.Get("followersCount").Int()),
+		Tags:        tags,
+		Cover:       extractor.ProxyUrl(categoryData.Get("avatarURL").String()),
+		Streams:     parsedStreams,
+	}
+
+	return parsedCategory, err
+}

extractor/twitch/Search.go

@@ -0,0 +1,103 @@
+package twitch
+
+import (
+	"safetwitch-backend/extractor"
+	"safetwitch-backend/extractor/structs"
+
+	"github.com/tidwall/gjson"
+)
+
+func GetSearchResult(query string) (structs.SearchResult, error) {
+	payload := []TwitchPayload{
+		{
+			"operationName": "SearchResultsPage_SearchResults",
+			"variables": map[string]interface{}{
+				"query":     query,
+				"options":   nil,
+				"requestID": "75948144-d051-4203-8511-57f3ee9b809a",
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "6ea6e6f66006485e41dbe3ebd69d5674c5b22896ce7b595d7fce6411a3790138",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return structs.SearchResult{}, err
+	}
+
+	rawStreamers := gjson.Get(string(body), "0.data.searchFor.channels.edges")
+	parsedStreamers := []structs.Streamer{}
+	for _, streamer := range rawStreamers.Array() {
+		stream := streamer.Get("item.stream").String()
+
+		parsedStreamers = append(parsedStreamers, structs.Streamer{
+			Username:  streamer.Get("item.login").String(),
+			Followers: int(streamer.Get("item.followers.totalCount").Int()),
+			IsLive:    !(stream == ""),
+			About:     streamer.Get("item.description").String(),
+			Pfp:       extractor.ProxyUrl(streamer.Get("item.profileImageURL").String()),
+			IsPartner: false,
+			ColorHex:  nil,
+			Id:        streamer.Get("item.channel.id").String(),
+		})
+	}
+
+	parsedCategories := []structs.CategoryPreview{}
+	rawCategories := gjson.Get(string(body), "0.data.searchFor.games.edges")
+
+	for _, category := range rawCategories.Array() {
+		var tags []string
+		for _, tag := range category.Get("item.tags").Array() {
+			tags = append(tags, tag.Get("tagName").String())
+		}
+
+		parsedCategories = append(parsedCategories, structs.CategoryPreview{
+			Name:        category.Get("item.name").String(),
+			DisplayName: category.Get("item.displayName").String(),
+			Viewers:     int(category.Get("item.viewersCount").Int()),
+			Image:       category.Get("item.boxArtURL").String(),
+			Tags:        tags,
+		})
+	}
+
+	foundRelatedLiveChannels := []structs.Streamer{}
+	streams := gjson.Get(string(body), "0.data.searchFor.relatedLiveChannels.edges")
+	for _, channel := range streams.Array() {
+		name := channel.Get("item.stream.broadcaster.login").String()
+		channel, err := GetStreamerInfo(name)
+		if err != nil {
+			return structs.SearchResult{}, nil
+		}
+		foundRelatedLiveChannels = append(foundRelatedLiveChannels, channel)
+	}
+
+	foundChannelsWithTag := []structs.Streamer{}
+	streams = gjson.Get(string(body), "0.data.searchFor.channelsWithTag.edges")
+	for _, stream := range streams.Array() {
+		foundChannelsWithTag = append(foundChannelsWithTag, structs.Streamer{
+			Username:  stream.Get("item.login").String(),
+			Followers: int(stream.Get("item.followers.totalCount").Int()),
+			IsLive:    !(stream.Get("stream").String() != ""),
+			About:     stream.Get("item.description").String(),
+			Pfp:       extractor.ProxyUrl(stream.Get("item.profileImageURL").String()),
+			IsPartner: false,
+			ColorHex:  nil,
+			Id:        stream.Get("item.channel.id").String(),
+		})
+	}
+
+	final := structs.SearchResult{
+		Channels:            parsedStreamers,
+		Categories:          parsedCategories,
+		RelatedLiveChannels: foundRelatedLiveChannels,
+		ChannelsWithTag:     foundChannelsWithTag,
+	}
+
+	return final, nil
+
+}

extractor/twitch/Stream.go

@@ -0,0 +1,91 @@
+package twitch
+
+import (
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/tidwall/gjson"
+)
+
+func GetStream(streamerName string) (string, error) {
+	// STAGE 1
+	// Get playback token from twitch
+	// Same request browser makes
+	payload1 := []TwitchPayload{
+		{
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712",
+				},
+			},
+			"operationName": "PlaybackAccessToken",
+			"variables": map[string]interface{}{
+				"isLive":     true,
+				"login":      streamerName,
+				"isVod":      false,
+				"vodID":      "",
+				"playerType": "site",
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload1)
+	if err != nil {
+		return "", err
+	}
+	// These will be needed to get playlist file from twitch
+	token := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.value").String()
+	signature := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.signature").String()
+	if err != nil {
+		return "", err
+	}
+
+	playlistUrl := "https://usher.ttvnw.net/api/channel/hls/" + strings.ToLower(streamerName) + ".m3u8"
+	params := "?sig=" + signature + "&token=" + token
+
+	req, err := http.NewRequest("GET", playlistUrl+params, nil)
+	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
+	if err != nil {
+		return "", err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+
+	playlistFile, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	// holy zooks, what the scallop??? we got the playlist, houston!!!
+	// time to proxy all the urls!!!
+	proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), false)
+	return proxiedPlaylist, nil
+
+}
+
+func GetSubPlaylist(rawurl string) (string, error) {
+	req, err := http.NewRequest("GET", rawurl, nil)
+
+	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
+	if err != nil {
+		return "", err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+
+	playlistFile, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), true)
+	return proxiedPlaylist, nil
+}

extractor/twitch/Streamer.go

@@ -0,0 +1,169 @@
+package twitch
+
+import (
+	"errors"
+	"safetwitch-backend/extractor/structs"
+
+	"github.com/tidwall/gjson"
+)
+
+func GetStreamerInfo(streamerName string) (structs.Streamer, error) {
+	payload := []TwitchPayload{
+		// Streamer data
+		{
+			"operationName": "ChannelRoot_AboutPanel",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+				"skipSchedule": false,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
+				},
+			},
+		},
+
+		// Stream metadata
+		{
+			"operationName": "StreamMetadata",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962",
+				},
+			},
+		},
+
+		// Stream tags
+		{
+			"operationName": "StreamTagsTrackingChannel",
+			"variables": map[string]interface{}{
+				"channel": streamerName,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "6aa3851aaaf88c320d514eb173563d430b28ed70fdaaf7eeef6ed4b812f48608",
+				},
+			},
+		},
+
+		// Stream preview image
+		{
+			"operationName": "VideoPreviewOverlay",
+			"variables": map[string]interface{}{
+				"login": streamerName,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f",
+				},
+			},
+		},
+
+		// Current views
+		{
+			"operationName": "UseViewCount",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "00b11c9c428f79ae228f30080a06ffd8226a1f068d6f52fbc057cbde66e994c2",
+				},
+			},
+		},
+
+		// Get offline banner image
+		{
+			"operationName": "ChannelShell",
+			"variables": map[string]interface{}{
+				"login": streamerName,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "580ab410bcd0c1ad194224957ae2241e5d252b2c5173d8e0cce9d32d5bb14efe",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return structs.Streamer{}, err
+	}
+
+	// begin parsing response
+	streamerFound := gjson.Get(string(body), "0.data.user")
+	if streamerFound.String() == "" {
+		return structs.Streamer{}, errors.New("streamer not found")
+	}
+
+	streamerData := gjson.Get(string(body), "0.data")
+	parsedSocials, err := ParseSocials(streamerData.String())
+	if err != nil {
+		return structs.Streamer{}, err
+	}
+
+	parsedStream, err := ParseStream(string(body))
+	var isLive bool
+	if err != nil {
+		if err.Error() == "streamer is not live" {
+			parsedStream = nil
+		} else {
+			return structs.Streamer{}, err
+		}
+	}
+	if parsedStream != nil {
+		isLive = true
+	} else {
+		isLive = false
+	}
+
+	streamerBanner := gjson.Get(string(body), "5.data.userOrError.bannerImageURL").String()
+
+	parsedStreamer, err := ParseStreamer(streamerData, isLive, parsedSocials, parsedStream, streamerName, streamerBanner)
+	if err != nil {
+		return structs.Streamer{}, err
+	}
+
+	return parsedStreamer, nil
+}
+
+func GetStreamerId(channelName string) (string, error) {
+	payload := []TwitchPayload{
+		{
+			"operationName": "ChannelRoot_AboutPanel",
+			"variables": map[string]interface{}{
+				"channelLogin": channelName,
+				"skipSchedule": true,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return "", err
+	}
+
+	id := gjson.Get(string(body), "0.data.user.id").String()
+
+	if id != "" {
+		return id, nil
+	} else {
+		return "", errors.New("could not find user")
+	}
+}

extractor/twitch/VODPreview.go

@@ -0,0 +1,56 @@
+package twitch
+
+import (
+	"safetwitch-backend/extractor/structs"
+
+	"github.com/tidwall/gjson"
+)
+
+/*
+If you don't understand the meaning of shelve in
+this context, go to an offline streamer on twitch
+and look under the videos tag. Each row is considered
+a shelve, and each can be expanded. This is to be used
+on a streamers profile page
+*/
+func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) {
+	payload := []TwitchPayload{
+		{
+			"operationName": "ChannelVideoShelvesQuery",
+			"variables": map[string]interface{}{
+				"channelLogin": channelName,
+				"first":        5,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "8afefb1ed16c4d8e20fa55024a7ed1727f63b6eca47d8d33a28500770bad8479",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return []structs.Shelve{}, err
+	}
+
+	/*
+		Twitch seperates videos by shelves.
+		There are 4 shells:
+		- Recent broadcasts
+		- Recent highlights and uploads
+		- Popular clips
+		- All videos
+
+		Each one of these shelves can be expanded to get more data
+	*/
+
+	shelves := gjson.Get(string(body), "0.data.user.videoShelves.edges")
+	parsedShelves := []structs.Shelve{}
+	for _, shelve := range shelves.Array() {
+		parsedShelves = append(parsedShelves, ParseShelve(shelve.Get("node")))
+	}
+
+	return parsedShelves, nil
+}

extractor/twitch/twitchExtractor.go

@@ -1,631 +0,0 @@
-package twitch
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"io"
-	"net/http"
-	"safetwitch-backend/extractor"
-	"safetwitch-backend/extractor/structs"
-	"strings"
-
-	"github.com/tidwall/gjson"
-)
-
-const twitchUrl = "https://gql.twitch.tv/gql"
-
-var language string = "en"
-
-// useful funcs
-func SetLanguage(code string) {
-	language = code
-}
-
-func call(url, method string, body io.Reader) (*http.Response, error) {
-	req, err := http.NewRequest(method, url, body)
-	if err != nil {
-		return req.Response, err
-	}
-
-	req.Header.Add("Accept-Language", language)
-	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
-	resp, err := http.DefaultClient.Do(req)
-	return resp, err
-}
-
-func parseResponse(payload []TwitchPayload) (response *[]structs.TwitchApiResponse, body []byte, err error) {
-	json_data, err := json.Marshal(payload)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	resp, err := call(twitchUrl, "POST", bytes.NewBuffer(json_data))
-	if err != nil {
-		return nil, nil, err
-	}
-	defer resp.Body.Close()
-
-	rawBody, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	var twitchResponse []structs.TwitchApiResponse
-	err = json.Unmarshal(rawBody, &twitchResponse)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return &twitchResponse, rawBody, nil
-}
-
-type TwitchPayload = map[string]interface{}
-
-func GetStreamerInfo(streamerName string) (structs.Streamer, error) {
-	payload := []TwitchPayload{
-		// Streamer data
-		{
-			"operationName": "ChannelRoot_AboutPanel",
-			"variables": map[string]interface{}{
-				"channelLogin": streamerName,
-				"skipSchedule": false,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
-				},
-			},
-		},
-
-		// Stream metadata
-		{
-			"operationName": "StreamMetadata",
-			"variables": map[string]interface{}{
-				"channelLogin": streamerName,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962",
-				},
-			},
-		},
-
-		// Stream tags
-		{
-			"operationName": "StreamTagsTrackingChannel",
-			"variables": map[string]interface{}{
-				"channel": streamerName,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "6aa3851aaaf88c320d514eb173563d430b28ed70fdaaf7eeef6ed4b812f48608",
-				},
-			},
-		},
-
-		// Stream preview image
-		{
-			"operationName": "VideoPreviewOverlay",
-			"variables": map[string]interface{}{
-				"login": streamerName,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f",
-				},
-			},
-		},
-
-		// Current views
-		{
-			"operationName": "UseViewCount",
-			"variables": map[string]interface{}{
-				"channelLogin": streamerName,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "00b11c9c428f79ae228f30080a06ffd8226a1f068d6f52fbc057cbde66e994c2",
-				},
-			},
-		},
-
-		// Get offline banner image
-		{
-			"operationName": "ChannelShell",
-			"variables": map[string]interface{}{
-				"login": streamerName,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "580ab410bcd0c1ad194224957ae2241e5d252b2c5173d8e0cce9d32d5bb14efe",
-				},
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return structs.Streamer{}, err
-	}
-
-	// begin parsing response
-	streamerFound := gjson.Get(string(body), "0.data.user")
-	if streamerFound.String() == "" {
-		return structs.Streamer{}, errors.New("streamer not found")
-	}
-
-	streamerData := gjson.Get(string(body), "0.data")
-	parsedSocials, err := ParseSocials(streamerData.String())
-	if err != nil {
-		return structs.Streamer{}, err
-	}
-
-	parsedStream, err := ParseStream(string(body))
-	var isLive bool
-	if err != nil {
-		if err.Error() == "streamer is not live" {
-			parsedStream = nil
-		} else {
-			return structs.Streamer{}, err
-		}
-	}
-	if parsedStream != nil {
-		isLive = true
-	} else {
-		isLive = false
-	}
-
-	streamerBanner := gjson.Get(string(body), "5.data.userOrError.bannerImageURL").String()
-
-	parsedStreamer, err := ParseStreamer(streamerData, isLive, parsedSocials, parsedStream, streamerName, streamerBanner)
-	if err != nil {
-		return structs.Streamer{}, err
-	}
-
-	return parsedStreamer, nil
-}
-
-func GetDiscoveryPage(limit int, cursor string) ([]structs.CategoryPreview, error) {
-	payload := []TwitchPayload{
-		{
-			"operationName": "BrowsePage_AllDirectories",
-			"variables": map[string]interface{}{
-				"limit": limit,
-				"options": map[string]interface{}{
-					"recommendationsContext": map[string]interface{}{
-						"platform": "web",
-					},
-					"requestID": "JIRA-VXP-2397",
-					"sort":      "RELEVANCE",
-					"tags":      []string{},
-				},
-				"cursor": cursor,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "1d1914ca3cbfaa607ecd5595b2e305e96acf987c8f25328f7713b25f604c4668",
-				},
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return []structs.CategoryPreview{}, err
-	}
-
-	var parsedCategoryArray []structs.CategoryPreview
-	categoryArray := gjson.Get(string(body), "0.data.directoriesWithTags.edges")
-	for _, categoryRes := range categoryArray.Array() {
-		parsedCategory, err := ParseCategory(categoryRes)
-		if err != nil {
-			return []structs.CategoryPreview{}, nil
-		}
-
-		parsedCategoryArray = append(parsedCategoryArray, parsedCategory)
-	}
-
-	return parsedCategoryArray, nil
-}
-
-func GetDiscoveryItem(name string, streamLimit int, cursor string) (structs.CategoryData, error) {
-	payload := []TwitchPayload{
-		{
-			"operationName": "DirectoryPage_Game",
-			"variables": map[string]interface{}{
-				"cursor":     cursor,
-				"imageWidth": 50,
-				"name":       name,
-				"options": map[string]interface{}{
-					"sort": "RELEVANCE",
-					"recommendationsContext": map[string]interface{}{
-						"platform": "web",
-					},
-					"requestID":    "JIRA-VXP-2397",
-					"freeformTags": nil,
-					"tags":         []string{},
-				},
-				"sortTypeIsRecency": false,
-				"limit":             streamLimit,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "df4bb6cc45055237bfaf3ead608bbafb79815c7100b6ee126719fac3762ddf8b",
-				},
-			},
-		},
-		{
-			"operationName": "Directory_DirectoryBanner",
-			"variables": map[string]interface{}{
-				"name": name,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "2670fbecd8fbea0211c56528d6eff5752ef9d6c73cd5238d395784b46335ded4",
-				},
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return structs.CategoryData{}, err
-	}
-
-	categoryStreams := gjson.Get(string(body), "0.data.game.streams.edges")
-
-	var parsedStreams []structs.CategoryMinifiedStream
-	for _, stream := range categoryStreams.Array() {
-		parsed, err := ParseMinifiedStream(stream)
-		if err != nil {
-			return structs.CategoryData{}, err
-		}
-		parsedStreams = append(parsedStreams, parsed)
-	}
-
-	categoryData := gjson.Get(string(body), "1.data.game")
-
-	var tags []string
-	for _, tag := range categoryData.Get("tags").Array() {
-		tags = append(tags, tag.Get("localizedName").String())
-	}
-
-	parsedCategory := structs.CategoryData{
-		Name:        categoryData.Get("name").String(),
-		DisplayName: categoryData.Get("displayName").String(),
-		Description: categoryData.Get("description").String(),
-		Viewers:     int(categoryData.Get("viewersCount").Int()),
-		Followers:   int(categoryData.Get("followersCount").Int()),
-		Tags:        tags,
-		Cover:       extractor.ProxyUrl(categoryData.Get("avatarURL").String()),
-		Streams:     parsedStreams,
-	}
-
-	return parsedCategory, err
-}
-
-func GetTwitchBadges() ([]structs.Badge, error) {
-	payload := []TwitchPayload{
-		{
-			"operationName": "ChannelPointsPredictionBadges",
-			"variables":     map[string]interface{}{},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"sha256Hash": "36995b30b22c31d1cd0aa329987ac9b5368bb7e6e1ab1df42808bdaa80a6dbf9",
-					"version":    1,
-				},
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return []structs.Badge{}, err
-	}
-
-	rawBadges := gjson.Get(string(body), "0.data.badges")
-	formattedBadges, err := ParseBadges(rawBadges)
-	if err != nil {
-		return []structs.Badge{}, err
-	}
-
-	return formattedBadges, nil
-}
-
-func GetStreamerBadges(streamerName string) ([]structs.Badge, error) {
-	payload := []TwitchPayload{
-		{
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"sha256Hash": "86f43113c04606e6476e39dcd432dee47c994d77a83e54b732e11d4935f0cd08",
-					"version":    1,
-				},
-			},
-			"operationName": "ChatList_Badges",
-			"variables": map[string]interface{}{
-				"channelLogin": streamerName,
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return []structs.Badge{}, err
-	}
-
-	rawBadges := gjson.Get(string(body), "0.data.user.broadcastBadges")
-
-	formattedBadges, err := ParseBadges(rawBadges)
-	if err != nil {
-		return []structs.Badge{}, err
-	}
-
-	return formattedBadges, nil
-}
-
-func GetStream(streamerName string) (string, error) {
-	// STAGE 1
-	// Get playback token from twitch
-	// Same request browser makes
-	payload1 := []TwitchPayload{
-		{
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712",
-				},
-			},
-			"operationName": "PlaybackAccessToken",
-			"variables": map[string]interface{}{
-				"isLive":     true,
-				"login":      streamerName,
-				"isVod":      false,
-				"vodID":      "",
-				"playerType": "site",
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload1)
-	if err != nil {
-		return "", err
-	}
-	// These will be needed to get playlist file from twitch
-	token := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.value").String()
-	signature := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.signature").String()
-	if err != nil {
-		return "", err
-	}
-
-	playlistUrl := "https://usher.ttvnw.net/api/channel/hls/" + strings.ToLower(streamerName) + ".m3u8"
-	params := "?sig=" + signature + "&token=" + token
-
-	req, err := http.NewRequest("GET", playlistUrl+params, nil)
-	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
-	if err != nil {
-		return "", err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-
-	playlistFile, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-
-	// holy zooks, what the scallop??? we got the playlist, houston!!!
-	// time to proxy all the urls!!!
-	proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), false)
-	return proxiedPlaylist, nil
-
-}
-
-func GetSubPlaylist(rawurl string) (string, error) {
-	req, err := http.NewRequest("GET", rawurl, nil)
-
-	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
-	if err != nil {
-		return "", err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-
-	playlistFile, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-
-	proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), true)
-	return proxiedPlaylist, nil
-}
-
-func GetSearchResult(query string) (structs.SearchResult, error) {
-	payload := []TwitchPayload{
-		{
-			"operationName": "SearchResultsPage_SearchResults",
-			"variables": map[string]interface{}{
-				"query":     query,
-				"options":   nil,
-				"requestID": "75948144-d051-4203-8511-57f3ee9b809a",
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "6ea6e6f66006485e41dbe3ebd69d5674c5b22896ce7b595d7fce6411a3790138",
-				},
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return structs.SearchResult{}, err
-	}
-
-	rawStreamers := gjson.Get(string(body), "0.data.searchFor.channels.edges")
-	parsedStreamers := []structs.Streamer{}
-	for _, streamer := range rawStreamers.Array() {
-		stream := streamer.Get("item.stream").String()
-
-		parsedStreamers = append(parsedStreamers, structs.Streamer{
-			Username:  streamer.Get("item.login").String(),
-			Followers: int(streamer.Get("item.followers.totalCount").Int()),
-			IsLive:    !(stream == ""),
-			About:     streamer.Get("item.description").String(),
-			Pfp:       extractor.ProxyUrl(streamer.Get("item.profileImageURL").String()),
-			IsPartner: false,
-			ColorHex:  nil,
-			Id:        streamer.Get("item.channel.id").String(),
-		})
-	}
-
-	parsedCategories := []structs.CategoryPreview{}
-	rawCategories := gjson.Get(string(body), "0.data.searchFor.games.edges")
-
-	for _, category := range rawCategories.Array() {
-		var tags []string
-		for _, tag := range category.Get("item.tags").Array() {
-			tags = append(tags, tag.Get("tagName").String())
-		}
-
-		parsedCategories = append(parsedCategories, structs.CategoryPreview{
-			Name:        category.Get("item.name").String(),
-			DisplayName: category.Get("item.displayName").String(),
-			Viewers:     int(category.Get("item.viewersCount").Int()),
-			Image:       category.Get("item.boxArtURL").String(),
-			Tags:        tags,
-		})
-	}
-
-	foundRelatedLiveChannels := []structs.Streamer{}
-	streams := gjson.Get(string(body), "0.data.searchFor.relatedLiveChannels.edges")
-	for _, channel := range streams.Array() {
-		name := channel.Get("item.stream.broadcaster.login").String()
-		channel, err := GetStreamerInfo(name)
-		if err != nil {
-			return structs.SearchResult{}, nil
-		}
-		foundRelatedLiveChannels = append(foundRelatedLiveChannels, channel)
-	}
-
-	foundChannelsWithTag := []structs.Streamer{}
-	streams = gjson.Get(string(body), "0.data.searchFor.channelsWithTag.edges")
-	for _, stream := range streams.Array() {
-		foundChannelsWithTag = append(foundChannelsWithTag, structs.Streamer{
-			Username:  stream.Get("item.login").String(),
-			Followers: int(stream.Get("item.followers.totalCount").Int()),
-			IsLive:    !(stream.Get("stream").String() != ""),
-			About:     stream.Get("item.description").String(),
-			Pfp:       extractor.ProxyUrl(stream.Get("item.profileImageURL").String()),
-			IsPartner: false,
-			ColorHex:  nil,
-			Id:        stream.Get("item.channel.id").String(),
-		})
-	}
-
-	final := structs.SearchResult{
-		Channels:            parsedStreamers,
-		Categories:          parsedCategories,
-		RelatedLiveChannels: foundRelatedLiveChannels,
-		ChannelsWithTag:     foundChannelsWithTag,
-	}
-
-	return final, nil
-
-}
-
-func GetStreamerId(channelName string) (string, error) {
-	payload := []TwitchPayload{
-		{
-			"operationName": "ChannelRoot_AboutPanel",
-			"variables": map[string]interface{}{
-				"channelLogin": channelName,
-				"skipSchedule": true,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
-				},
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return "", err
-	}
-
-	id := gjson.Get(string(body), "0.data.user.id").String()
-
-	if id != "" {
-		return id, nil
-	} else {
-		return "", errors.New("could not find user")
-	}
-}
-
-/*
-If you don't understand the meaning of shelve in
-this context, go to an offline streamer on twitch
-and look under the videos tag. Each row is considered
-a shelve, and each can be expanded. This is to be used
-on a streamers profile page
-*/
-func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) {
-	payload := []TwitchPayload{
-		{
-			"operationName": "ChannelVideoShelvesQuery",
-			"variables": map[string]interface{}{
-				"channelLogin": channelName,
-				"first":        5,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "8afefb1ed16c4d8e20fa55024a7ed1727f63b6eca47d8d33a28500770bad8479",
-				},
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return []structs.Shelve{}, err
-	}
-
-	/*
-		Twitch seperates videos by shelves.
-		There are 4 shells:
-		- Recent broadcasts
-		- Recent highlights and uploads
-		- Popular clips
-		- All videos
-
-		Each one of these shelves can be expanded to get more data
-	*/
-
-	shelves := gjson.Get(string(body), "0.data.user.videoShelves.edges")
-	parsedShelves := []structs.Shelve{}
-	for _, shelve := range shelves.Array() {
-		parsedShelves = append(parsedShelves, ParseShelve(shelve.Get("node")))
-	}
-
-	return parsedShelves, nil
-}

extractor/twitch/utils.go

@@ -0,0 +1,58 @@
+package twitch
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"net/http"
+	"safetwitch-backend/extractor/structs"
+)
+
+const twitchUrl = "https://gql.twitch.tv/gql"
+
+var Language string = "en"
+
+// useful funcs
+func SetLanguage(code string) {
+	Language = code
+}
+
+func call(url, method string, body io.Reader) (*http.Response, error) {
+	req, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return req.Response, err
+	}
+
+	req.Header.Add("Accept-Language", Language)
+	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
+	resp, err := http.DefaultClient.Do(req)
+	return resp, err
+}
+
+func parseResponse(payload []TwitchPayload) (response *[]structs.TwitchApiResponse, body []byte, err error) {
+	json_data, err := json.Marshal(payload)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	resp, err := call(twitchUrl, "POST", bytes.NewBuffer(json_data))
+	if err != nil {
+		return nil, nil, err
+	}
+	defer resp.Body.Close()
+
+	rawBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var twitchResponse []structs.TwitchApiResponse
+	err = json.Unmarshal(rawBody, &twitchResponse)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return &twitchResponse, rawBody, nil
+}
+
+type TwitchPayload = map[string]interface{}