From 87dc0b8d2caf78e9fe287161484f5f2be6e1ec54 Mon Sep 17 00:00:00 2001 From: dragongoose Date: Sun, 16 Jul 2023 22:03:52 -0400 Subject: [PATCH] Split fetchers into multiple files for readability --- extractor/twitch/Badges.go | 66 +++ extractor/twitch/Discover.go | 130 ++++++ extractor/twitch/Search.go | 103 +++++ extractor/twitch/Stream.go | 91 ++++ extractor/twitch/Streamer.go | 169 ++++++++ extractor/twitch/VODPreview.go | 56 +++ extractor/twitch/twitchExtractor.go | 631 ---------------------------- extractor/twitch/utils.go | 58 +++ 8 files changed, 673 insertions(+), 631 deletions(-) create mode 100644 extractor/twitch/Badges.go create mode 100644 extractor/twitch/Discover.go create mode 100644 extractor/twitch/Search.go create mode 100644 extractor/twitch/Stream.go create mode 100644 extractor/twitch/Streamer.go create mode 100644 extractor/twitch/VODPreview.go delete mode 100644 extractor/twitch/twitchExtractor.go create mode 100644 extractor/twitch/utils.go diff --git a/extractor/twitch/Badges.go b/extractor/twitch/Badges.go new file mode 100644 index 0000000..8617734 --- /dev/null +++ b/extractor/twitch/Badges.go @@ -0,0 +1,66 @@ +package twitch + +import ( + "safetwitch-backend/extractor/structs" + + "github.com/tidwall/gjson" +) + +func GetTwitchBadges() ([]structs.Badge, error) { + payload := []TwitchPayload{ + { + "operationName": "ChannelPointsPredictionBadges", + "variables": map[string]interface{}{}, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "sha256Hash": "36995b30b22c31d1cd0aa329987ac9b5368bb7e6e1ab1df42808bdaa80a6dbf9", + "version": 1, + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return []structs.Badge{}, err + } + + rawBadges := gjson.Get(string(body), "0.data.badges") + formattedBadges, err := ParseBadges(rawBadges) + if err != nil { + return []structs.Badge{}, err + } + + return formattedBadges, nil +} + +func GetStreamerBadges(streamerName string) ([]structs.Badge, error) { + payload := []TwitchPayload{ + { + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "sha256Hash": "86f43113c04606e6476e39dcd432dee47c994d77a83e54b732e11d4935f0cd08", + "version": 1, + }, + }, + "operationName": "ChatList_Badges", + "variables": map[string]interface{}{ + "channelLogin": streamerName, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return []structs.Badge{}, err + } + + rawBadges := gjson.Get(string(body), "0.data.user.broadcastBadges") + + formattedBadges, err := ParseBadges(rawBadges) + if err != nil { + return []structs.Badge{}, err + } + + return formattedBadges, nil +} diff --git a/extractor/twitch/Discover.go b/extractor/twitch/Discover.go new file mode 100644 index 0000000..1db98b5 --- /dev/null +++ b/extractor/twitch/Discover.go @@ -0,0 +1,130 @@ +package twitch + +import ( + "safetwitch-backend/extractor" + "safetwitch-backend/extractor/structs" + + "github.com/tidwall/gjson" +) + +func GetDiscoveryPage(limit int, cursor string) ([]structs.CategoryPreview, error) { + payload := []TwitchPayload{ + { + "operationName": "BrowsePage_AllDirectories", + "variables": map[string]interface{}{ + "limit": limit, + "options": map[string]interface{}{ + "recommendationsContext": map[string]interface{}{ + "platform": "web", + }, + "requestID": "JIRA-VXP-2397", + "sort": "RELEVANCE", + "tags": []string{}, + }, + "cursor": cursor, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "1d1914ca3cbfaa607ecd5595b2e305e96acf987c8f25328f7713b25f604c4668", + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return []structs.CategoryPreview{}, err + } + + var parsedCategoryArray []structs.CategoryPreview + categoryArray := gjson.Get(string(body), "0.data.directoriesWithTags.edges") + for _, categoryRes := range categoryArray.Array() { + parsedCategory, err := ParseCategory(categoryRes) + if err != nil { + return []structs.CategoryPreview{}, nil + } + + parsedCategoryArray = append(parsedCategoryArray, parsedCategory) + } + + return parsedCategoryArray, nil +} + +func GetDiscoveryItem(name string, streamLimit int, cursor string) (structs.CategoryData, error) { + payload := []TwitchPayload{ + { + "operationName": "DirectoryPage_Game", + "variables": map[string]interface{}{ + "cursor": cursor, + "imageWidth": 50, + "name": name, + "options": map[string]interface{}{ + "sort": "RELEVANCE", + "recommendationsContext": map[string]interface{}{ + "platform": "web", + }, + "requestID": "JIRA-VXP-2397", + "freeformTags": nil, + "tags": []string{}, + }, + "sortTypeIsRecency": false, + "limit": streamLimit, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "df4bb6cc45055237bfaf3ead608bbafb79815c7100b6ee126719fac3762ddf8b", + }, + }, + }, + { + "operationName": "Directory_DirectoryBanner", + "variables": map[string]interface{}{ + "name": name, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "2670fbecd8fbea0211c56528d6eff5752ef9d6c73cd5238d395784b46335ded4", + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return structs.CategoryData{}, err + } + + categoryStreams := gjson.Get(string(body), "0.data.game.streams.edges") + + var parsedStreams []structs.CategoryMinifiedStream + for _, stream := range categoryStreams.Array() { + parsed, err := ParseMinifiedStream(stream) + if err != nil { + return structs.CategoryData{}, err + } + parsedStreams = append(parsedStreams, parsed) + } + + categoryData := gjson.Get(string(body), "1.data.game") + + var tags []string + for _, tag := range categoryData.Get("tags").Array() { + tags = append(tags, tag.Get("localizedName").String()) + } + + parsedCategory := structs.CategoryData{ + Name: categoryData.Get("name").String(), + DisplayName: categoryData.Get("displayName").String(), + Description: categoryData.Get("description").String(), + Viewers: int(categoryData.Get("viewersCount").Int()), + Followers: int(categoryData.Get("followersCount").Int()), + Tags: tags, + Cover: extractor.ProxyUrl(categoryData.Get("avatarURL").String()), + Streams: parsedStreams, + } + + return parsedCategory, err +} diff --git a/extractor/twitch/Search.go b/extractor/twitch/Search.go new file mode 100644 index 0000000..1b24801 --- /dev/null +++ b/extractor/twitch/Search.go @@ -0,0 +1,103 @@ +package twitch + +import ( + "safetwitch-backend/extractor" + "safetwitch-backend/extractor/structs" + + "github.com/tidwall/gjson" +) + +func GetSearchResult(query string) (structs.SearchResult, error) { + payload := []TwitchPayload{ + { + "operationName": "SearchResultsPage_SearchResults", + "variables": map[string]interface{}{ + "query": query, + "options": nil, + "requestID": "75948144-d051-4203-8511-57f3ee9b809a", + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "6ea6e6f66006485e41dbe3ebd69d5674c5b22896ce7b595d7fce6411a3790138", + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return structs.SearchResult{}, err + } + + rawStreamers := gjson.Get(string(body), "0.data.searchFor.channels.edges") + parsedStreamers := []structs.Streamer{} + for _, streamer := range rawStreamers.Array() { + stream := streamer.Get("item.stream").String() + + parsedStreamers = append(parsedStreamers, structs.Streamer{ + Username: streamer.Get("item.login").String(), + Followers: int(streamer.Get("item.followers.totalCount").Int()), + IsLive: !(stream == ""), + About: streamer.Get("item.description").String(), + Pfp: extractor.ProxyUrl(streamer.Get("item.profileImageURL").String()), + IsPartner: false, + ColorHex: nil, + Id: streamer.Get("item.channel.id").String(), + }) + } + + parsedCategories := []structs.CategoryPreview{} + rawCategories := gjson.Get(string(body), "0.data.searchFor.games.edges") + + for _, category := range rawCategories.Array() { + var tags []string + for _, tag := range category.Get("item.tags").Array() { + tags = append(tags, tag.Get("tagName").String()) + } + + parsedCategories = append(parsedCategories, structs.CategoryPreview{ + Name: category.Get("item.name").String(), + DisplayName: category.Get("item.displayName").String(), + Viewers: int(category.Get("item.viewersCount").Int()), + Image: category.Get("item.boxArtURL").String(), + Tags: tags, + }) + } + + foundRelatedLiveChannels := []structs.Streamer{} + streams := gjson.Get(string(body), "0.data.searchFor.relatedLiveChannels.edges") + for _, channel := range streams.Array() { + name := channel.Get("item.stream.broadcaster.login").String() + channel, err := GetStreamerInfo(name) + if err != nil { + return structs.SearchResult{}, nil + } + foundRelatedLiveChannels = append(foundRelatedLiveChannels, channel) + } + + foundChannelsWithTag := []structs.Streamer{} + streams = gjson.Get(string(body), "0.data.searchFor.channelsWithTag.edges") + for _, stream := range streams.Array() { + foundChannelsWithTag = append(foundChannelsWithTag, structs.Streamer{ + Username: stream.Get("item.login").String(), + Followers: int(stream.Get("item.followers.totalCount").Int()), + IsLive: !(stream.Get("stream").String() != ""), + About: stream.Get("item.description").String(), + Pfp: extractor.ProxyUrl(stream.Get("item.profileImageURL").String()), + IsPartner: false, + ColorHex: nil, + Id: stream.Get("item.channel.id").String(), + }) + } + + final := structs.SearchResult{ + Channels: parsedStreamers, + Categories: parsedCategories, + RelatedLiveChannels: foundRelatedLiveChannels, + ChannelsWithTag: foundChannelsWithTag, + } + + return final, nil + +} diff --git a/extractor/twitch/Stream.go b/extractor/twitch/Stream.go new file mode 100644 index 0000000..78dbaa6 --- /dev/null +++ b/extractor/twitch/Stream.go @@ -0,0 +1,91 @@ +package twitch + +import ( + "io" + "net/http" + "strings" + + "github.com/tidwall/gjson" +) + +func GetStream(streamerName string) (string, error) { + // STAGE 1 + // Get playback token from twitch + // Same request browser makes + payload1 := []TwitchPayload{ + { + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712", + }, + }, + "operationName": "PlaybackAccessToken", + "variables": map[string]interface{}{ + "isLive": true, + "login": streamerName, + "isVod": false, + "vodID": "", + "playerType": "site", + }, + }, + } + + _, body, err := parseResponse(payload1) + if err != nil { + return "", err + } + // These will be needed to get playlist file from twitch + token := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.value").String() + signature := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.signature").String() + if err != nil { + return "", err + } + + playlistUrl := "https://usher.ttvnw.net/api/channel/hls/" + strings.ToLower(streamerName) + ".m3u8" + params := "?sig=" + signature + "&token=" + token + + req, err := http.NewRequest("GET", playlistUrl+params, nil) + req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") + if err != nil { + return "", err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + + playlistFile, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + // holy zooks, what the scallop??? we got the playlist, houston!!! + // time to proxy all the urls!!! + proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), false) + return proxiedPlaylist, nil + +} + +func GetSubPlaylist(rawurl string) (string, error) { + req, err := http.NewRequest("GET", rawurl, nil) + + req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") + if err != nil { + return "", err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + + playlistFile, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), true) + return proxiedPlaylist, nil +} diff --git a/extractor/twitch/Streamer.go b/extractor/twitch/Streamer.go new file mode 100644 index 0000000..8f5d2e2 --- /dev/null +++ b/extractor/twitch/Streamer.go @@ -0,0 +1,169 @@ +package twitch + +import ( + "errors" + "safetwitch-backend/extractor/structs" + + "github.com/tidwall/gjson" +) + +func GetStreamerInfo(streamerName string) (structs.Streamer, error) { + payload := []TwitchPayload{ + // Streamer data + { + "operationName": "ChannelRoot_AboutPanel", + "variables": map[string]interface{}{ + "channelLogin": streamerName, + "skipSchedule": false, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6", + }, + }, + }, + + // Stream metadata + { + "operationName": "StreamMetadata", + "variables": map[string]interface{}{ + "channelLogin": streamerName, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962", + }, + }, + }, + + // Stream tags + { + "operationName": "StreamTagsTrackingChannel", + "variables": map[string]interface{}{ + "channel": streamerName, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "6aa3851aaaf88c320d514eb173563d430b28ed70fdaaf7eeef6ed4b812f48608", + }, + }, + }, + + // Stream preview image + { + "operationName": "VideoPreviewOverlay", + "variables": map[string]interface{}{ + "login": streamerName, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f", + }, + }, + }, + + // Current views + { + "operationName": "UseViewCount", + "variables": map[string]interface{}{ + "channelLogin": streamerName, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "00b11c9c428f79ae228f30080a06ffd8226a1f068d6f52fbc057cbde66e994c2", + }, + }, + }, + + // Get offline banner image + { + "operationName": "ChannelShell", + "variables": map[string]interface{}{ + "login": streamerName, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "580ab410bcd0c1ad194224957ae2241e5d252b2c5173d8e0cce9d32d5bb14efe", + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return structs.Streamer{}, err + } + + // begin parsing response + streamerFound := gjson.Get(string(body), "0.data.user") + if streamerFound.String() == "" { + return structs.Streamer{}, errors.New("streamer not found") + } + + streamerData := gjson.Get(string(body), "0.data") + parsedSocials, err := ParseSocials(streamerData.String()) + if err != nil { + return structs.Streamer{}, err + } + + parsedStream, err := ParseStream(string(body)) + var isLive bool + if err != nil { + if err.Error() == "streamer is not live" { + parsedStream = nil + } else { + return structs.Streamer{}, err + } + } + if parsedStream != nil { + isLive = true + } else { + isLive = false + } + + streamerBanner := gjson.Get(string(body), "5.data.userOrError.bannerImageURL").String() + + parsedStreamer, err := ParseStreamer(streamerData, isLive, parsedSocials, parsedStream, streamerName, streamerBanner) + if err != nil { + return structs.Streamer{}, err + } + + return parsedStreamer, nil +} + +func GetStreamerId(channelName string) (string, error) { + payload := []TwitchPayload{ + { + "operationName": "ChannelRoot_AboutPanel", + "variables": map[string]interface{}{ + "channelLogin": channelName, + "skipSchedule": true, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6", + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return "", err + } + + id := gjson.Get(string(body), "0.data.user.id").String() + + if id != "" { + return id, nil + } else { + return "", errors.New("could not find user") + } +} diff --git a/extractor/twitch/VODPreview.go b/extractor/twitch/VODPreview.go new file mode 100644 index 0000000..ad9b096 --- /dev/null +++ b/extractor/twitch/VODPreview.go @@ -0,0 +1,56 @@ +package twitch + +import ( + "safetwitch-backend/extractor/structs" + + "github.com/tidwall/gjson" +) + +/* +If you don't understand the meaning of shelve in +this context, go to an offline streamer on twitch +and look under the videos tag. Each row is considered +a shelve, and each can be expanded. This is to be used +on a streamers profile page +*/ +func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) { + payload := []TwitchPayload{ + { + "operationName": "ChannelVideoShelvesQuery", + "variables": map[string]interface{}{ + "channelLogin": channelName, + "first": 5, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "8afefb1ed16c4d8e20fa55024a7ed1727f63b6eca47d8d33a28500770bad8479", + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return []structs.Shelve{}, err + } + + /* + Twitch seperates videos by shelves. + There are 4 shells: + - Recent broadcasts + - Recent highlights and uploads + - Popular clips + - All videos + + Each one of these shelves can be expanded to get more data + */ + + shelves := gjson.Get(string(body), "0.data.user.videoShelves.edges") + parsedShelves := []structs.Shelve{} + for _, shelve := range shelves.Array() { + parsedShelves = append(parsedShelves, ParseShelve(shelve.Get("node"))) + } + + return parsedShelves, nil +} diff --git a/extractor/twitch/twitchExtractor.go b/extractor/twitch/twitchExtractor.go deleted file mode 100644 index 098c516..0000000 --- a/extractor/twitch/twitchExtractor.go +++ /dev/null @@ -1,631 +0,0 @@ -package twitch - -import ( - "bytes" - "encoding/json" - "errors" - "io" - "net/http" - "safetwitch-backend/extractor" - "safetwitch-backend/extractor/structs" - "strings" - - "github.com/tidwall/gjson" -) - -const twitchUrl = "https://gql.twitch.tv/gql" - -var language string = "en" - -// useful funcs -func SetLanguage(code string) { - language = code -} - -func call(url, method string, body io.Reader) (*http.Response, error) { - req, err := http.NewRequest(method, url, body) - if err != nil { - return req.Response, err - } - - req.Header.Add("Accept-Language", language) - req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") - resp, err := http.DefaultClient.Do(req) - return resp, err -} - -func parseResponse(payload []TwitchPayload) (response *[]structs.TwitchApiResponse, body []byte, err error) { - json_data, err := json.Marshal(payload) - if err != nil { - return nil, nil, err - } - - resp, err := call(twitchUrl, "POST", bytes.NewBuffer(json_data)) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - - rawBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, nil, err - } - - var twitchResponse []structs.TwitchApiResponse - err = json.Unmarshal(rawBody, &twitchResponse) - if err != nil { - return nil, nil, err - } - - return &twitchResponse, rawBody, nil -} - -type TwitchPayload = map[string]interface{} - -func GetStreamerInfo(streamerName string) (structs.Streamer, error) { - payload := []TwitchPayload{ - // Streamer data - { - "operationName": "ChannelRoot_AboutPanel", - "variables": map[string]interface{}{ - "channelLogin": streamerName, - "skipSchedule": false, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6", - }, - }, - }, - - // Stream metadata - { - "operationName": "StreamMetadata", - "variables": map[string]interface{}{ - "channelLogin": streamerName, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962", - }, - }, - }, - - // Stream tags - { - "operationName": "StreamTagsTrackingChannel", - "variables": map[string]interface{}{ - "channel": streamerName, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "6aa3851aaaf88c320d514eb173563d430b28ed70fdaaf7eeef6ed4b812f48608", - }, - }, - }, - - // Stream preview image - { - "operationName": "VideoPreviewOverlay", - "variables": map[string]interface{}{ - "login": streamerName, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f", - }, - }, - }, - - // Current views - { - "operationName": "UseViewCount", - "variables": map[string]interface{}{ - "channelLogin": streamerName, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "00b11c9c428f79ae228f30080a06ffd8226a1f068d6f52fbc057cbde66e994c2", - }, - }, - }, - - // Get offline banner image - { - "operationName": "ChannelShell", - "variables": map[string]interface{}{ - "login": streamerName, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "580ab410bcd0c1ad194224957ae2241e5d252b2c5173d8e0cce9d32d5bb14efe", - }, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return structs.Streamer{}, err - } - - // begin parsing response - streamerFound := gjson.Get(string(body), "0.data.user") - if streamerFound.String() == "" { - return structs.Streamer{}, errors.New("streamer not found") - } - - streamerData := gjson.Get(string(body), "0.data") - parsedSocials, err := ParseSocials(streamerData.String()) - if err != nil { - return structs.Streamer{}, err - } - - parsedStream, err := ParseStream(string(body)) - var isLive bool - if err != nil { - if err.Error() == "streamer is not live" { - parsedStream = nil - } else { - return structs.Streamer{}, err - } - } - if parsedStream != nil { - isLive = true - } else { - isLive = false - } - - streamerBanner := gjson.Get(string(body), "5.data.userOrError.bannerImageURL").String() - - parsedStreamer, err := ParseStreamer(streamerData, isLive, parsedSocials, parsedStream, streamerName, streamerBanner) - if err != nil { - return structs.Streamer{}, err - } - - return parsedStreamer, nil -} - -func GetDiscoveryPage(limit int, cursor string) ([]structs.CategoryPreview, error) { - payload := []TwitchPayload{ - { - "operationName": "BrowsePage_AllDirectories", - "variables": map[string]interface{}{ - "limit": limit, - "options": map[string]interface{}{ - "recommendationsContext": map[string]interface{}{ - "platform": "web", - }, - "requestID": "JIRA-VXP-2397", - "sort": "RELEVANCE", - "tags": []string{}, - }, - "cursor": cursor, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "1d1914ca3cbfaa607ecd5595b2e305e96acf987c8f25328f7713b25f604c4668", - }, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return []structs.CategoryPreview{}, err - } - - var parsedCategoryArray []structs.CategoryPreview - categoryArray := gjson.Get(string(body), "0.data.directoriesWithTags.edges") - for _, categoryRes := range categoryArray.Array() { - parsedCategory, err := ParseCategory(categoryRes) - if err != nil { - return []structs.CategoryPreview{}, nil - } - - parsedCategoryArray = append(parsedCategoryArray, parsedCategory) - } - - return parsedCategoryArray, nil -} - -func GetDiscoveryItem(name string, streamLimit int, cursor string) (structs.CategoryData, error) { - payload := []TwitchPayload{ - { - "operationName": "DirectoryPage_Game", - "variables": map[string]interface{}{ - "cursor": cursor, - "imageWidth": 50, - "name": name, - "options": map[string]interface{}{ - "sort": "RELEVANCE", - "recommendationsContext": map[string]interface{}{ - "platform": "web", - }, - "requestID": "JIRA-VXP-2397", - "freeformTags": nil, - "tags": []string{}, - }, - "sortTypeIsRecency": false, - "limit": streamLimit, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "df4bb6cc45055237bfaf3ead608bbafb79815c7100b6ee126719fac3762ddf8b", - }, - }, - }, - { - "operationName": "Directory_DirectoryBanner", - "variables": map[string]interface{}{ - "name": name, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "2670fbecd8fbea0211c56528d6eff5752ef9d6c73cd5238d395784b46335ded4", - }, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return structs.CategoryData{}, err - } - - categoryStreams := gjson.Get(string(body), "0.data.game.streams.edges") - - var parsedStreams []structs.CategoryMinifiedStream - for _, stream := range categoryStreams.Array() { - parsed, err := ParseMinifiedStream(stream) - if err != nil { - return structs.CategoryData{}, err - } - parsedStreams = append(parsedStreams, parsed) - } - - categoryData := gjson.Get(string(body), "1.data.game") - - var tags []string - for _, tag := range categoryData.Get("tags").Array() { - tags = append(tags, tag.Get("localizedName").String()) - } - - parsedCategory := structs.CategoryData{ - Name: categoryData.Get("name").String(), - DisplayName: categoryData.Get("displayName").String(), - Description: categoryData.Get("description").String(), - Viewers: int(categoryData.Get("viewersCount").Int()), - Followers: int(categoryData.Get("followersCount").Int()), - Tags: tags, - Cover: extractor.ProxyUrl(categoryData.Get("avatarURL").String()), - Streams: parsedStreams, - } - - return parsedCategory, err -} - -func GetTwitchBadges() ([]structs.Badge, error) { - payload := []TwitchPayload{ - { - "operationName": "ChannelPointsPredictionBadges", - "variables": map[string]interface{}{}, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "sha256Hash": "36995b30b22c31d1cd0aa329987ac9b5368bb7e6e1ab1df42808bdaa80a6dbf9", - "version": 1, - }, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return []structs.Badge{}, err - } - - rawBadges := gjson.Get(string(body), "0.data.badges") - formattedBadges, err := ParseBadges(rawBadges) - if err != nil { - return []structs.Badge{}, err - } - - return formattedBadges, nil -} - -func GetStreamerBadges(streamerName string) ([]structs.Badge, error) { - payload := []TwitchPayload{ - { - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "sha256Hash": "86f43113c04606e6476e39dcd432dee47c994d77a83e54b732e11d4935f0cd08", - "version": 1, - }, - }, - "operationName": "ChatList_Badges", - "variables": map[string]interface{}{ - "channelLogin": streamerName, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return []structs.Badge{}, err - } - - rawBadges := gjson.Get(string(body), "0.data.user.broadcastBadges") - - formattedBadges, err := ParseBadges(rawBadges) - if err != nil { - return []structs.Badge{}, err - } - - return formattedBadges, nil -} - -func GetStream(streamerName string) (string, error) { - // STAGE 1 - // Get playback token from twitch - // Same request browser makes - payload1 := []TwitchPayload{ - { - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712", - }, - }, - "operationName": "PlaybackAccessToken", - "variables": map[string]interface{}{ - "isLive": true, - "login": streamerName, - "isVod": false, - "vodID": "", - "playerType": "site", - }, - }, - } - - _, body, err := parseResponse(payload1) - if err != nil { - return "", err - } - // These will be needed to get playlist file from twitch - token := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.value").String() - signature := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.signature").String() - if err != nil { - return "", err - } - - playlistUrl := "https://usher.ttvnw.net/api/channel/hls/" + strings.ToLower(streamerName) + ".m3u8" - params := "?sig=" + signature + "&token=" + token - - req, err := http.NewRequest("GET", playlistUrl+params, nil) - req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") - if err != nil { - return "", err - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - - playlistFile, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - - // holy zooks, what the scallop??? we got the playlist, houston!!! - // time to proxy all the urls!!! - proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), false) - return proxiedPlaylist, nil - -} - -func GetSubPlaylist(rawurl string) (string, error) { - req, err := http.NewRequest("GET", rawurl, nil) - - req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") - if err != nil { - return "", err - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - - playlistFile, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - - proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), true) - return proxiedPlaylist, nil -} - -func GetSearchResult(query string) (structs.SearchResult, error) { - payload := []TwitchPayload{ - { - "operationName": "SearchResultsPage_SearchResults", - "variables": map[string]interface{}{ - "query": query, - "options": nil, - "requestID": "75948144-d051-4203-8511-57f3ee9b809a", - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "6ea6e6f66006485e41dbe3ebd69d5674c5b22896ce7b595d7fce6411a3790138", - }, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return structs.SearchResult{}, err - } - - rawStreamers := gjson.Get(string(body), "0.data.searchFor.channels.edges") - parsedStreamers := []structs.Streamer{} - for _, streamer := range rawStreamers.Array() { - stream := streamer.Get("item.stream").String() - - parsedStreamers = append(parsedStreamers, structs.Streamer{ - Username: streamer.Get("item.login").String(), - Followers: int(streamer.Get("item.followers.totalCount").Int()), - IsLive: !(stream == ""), - About: streamer.Get("item.description").String(), - Pfp: extractor.ProxyUrl(streamer.Get("item.profileImageURL").String()), - IsPartner: false, - ColorHex: nil, - Id: streamer.Get("item.channel.id").String(), - }) - } - - parsedCategories := []structs.CategoryPreview{} - rawCategories := gjson.Get(string(body), "0.data.searchFor.games.edges") - - for _, category := range rawCategories.Array() { - var tags []string - for _, tag := range category.Get("item.tags").Array() { - tags = append(tags, tag.Get("tagName").String()) - } - - parsedCategories = append(parsedCategories, structs.CategoryPreview{ - Name: category.Get("item.name").String(), - DisplayName: category.Get("item.displayName").String(), - Viewers: int(category.Get("item.viewersCount").Int()), - Image: category.Get("item.boxArtURL").String(), - Tags: tags, - }) - } - - foundRelatedLiveChannels := []structs.Streamer{} - streams := gjson.Get(string(body), "0.data.searchFor.relatedLiveChannels.edges") - for _, channel := range streams.Array() { - name := channel.Get("item.stream.broadcaster.login").String() - channel, err := GetStreamerInfo(name) - if err != nil { - return structs.SearchResult{}, nil - } - foundRelatedLiveChannels = append(foundRelatedLiveChannels, channel) - } - - foundChannelsWithTag := []structs.Streamer{} - streams = gjson.Get(string(body), "0.data.searchFor.channelsWithTag.edges") - for _, stream := range streams.Array() { - foundChannelsWithTag = append(foundChannelsWithTag, structs.Streamer{ - Username: stream.Get("item.login").String(), - Followers: int(stream.Get("item.followers.totalCount").Int()), - IsLive: !(stream.Get("stream").String() != ""), - About: stream.Get("item.description").String(), - Pfp: extractor.ProxyUrl(stream.Get("item.profileImageURL").String()), - IsPartner: false, - ColorHex: nil, - Id: stream.Get("item.channel.id").String(), - }) - } - - final := structs.SearchResult{ - Channels: parsedStreamers, - Categories: parsedCategories, - RelatedLiveChannels: foundRelatedLiveChannels, - ChannelsWithTag: foundChannelsWithTag, - } - - return final, nil - -} - -func GetStreamerId(channelName string) (string, error) { - payload := []TwitchPayload{ - { - "operationName": "ChannelRoot_AboutPanel", - "variables": map[string]interface{}{ - "channelLogin": channelName, - "skipSchedule": true, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6", - }, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return "", err - } - - id := gjson.Get(string(body), "0.data.user.id").String() - - if id != "" { - return id, nil - } else { - return "", errors.New("could not find user") - } -} - -/* -If you don't understand the meaning of shelve in -this context, go to an offline streamer on twitch -and look under the videos tag. Each row is considered -a shelve, and each can be expanded. This is to be used -on a streamers profile page -*/ -func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) { - payload := []TwitchPayload{ - { - "operationName": "ChannelVideoShelvesQuery", - "variables": map[string]interface{}{ - "channelLogin": channelName, - "first": 5, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "8afefb1ed16c4d8e20fa55024a7ed1727f63b6eca47d8d33a28500770bad8479", - }, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return []structs.Shelve{}, err - } - - /* - Twitch seperates videos by shelves. - There are 4 shells: - - Recent broadcasts - - Recent highlights and uploads - - Popular clips - - All videos - - Each one of these shelves can be expanded to get more data - */ - - shelves := gjson.Get(string(body), "0.data.user.videoShelves.edges") - parsedShelves := []structs.Shelve{} - for _, shelve := range shelves.Array() { - parsedShelves = append(parsedShelves, ParseShelve(shelve.Get("node"))) - } - - return parsedShelves, nil -} diff --git a/extractor/twitch/utils.go b/extractor/twitch/utils.go new file mode 100644 index 0000000..4c04138 --- /dev/null +++ b/extractor/twitch/utils.go @@ -0,0 +1,58 @@ +package twitch + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "safetwitch-backend/extractor/structs" +) + +const twitchUrl = "https://gql.twitch.tv/gql" + +var Language string = "en" + +// useful funcs +func SetLanguage(code string) { + Language = code +} + +func call(url, method string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequest(method, url, body) + if err != nil { + return req.Response, err + } + + req.Header.Add("Accept-Language", Language) + req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") + resp, err := http.DefaultClient.Do(req) + return resp, err +} + +func parseResponse(payload []TwitchPayload) (response *[]structs.TwitchApiResponse, body []byte, err error) { + json_data, err := json.Marshal(payload) + if err != nil { + return nil, nil, err + } + + resp, err := call(twitchUrl, "POST", bytes.NewBuffer(json_data)) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + rawBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, err + } + + var twitchResponse []structs.TwitchApiResponse + err = json.Unmarshal(rawBody, &twitchResponse) + if err != nil { + return nil, nil, err + } + + return &twitchResponse, rawBody, nil +} + +type TwitchPayload = map[string]interface{}