From 87dc0b8d2caf78e9fe287161484f5f2be6e1ec54 Mon Sep 17 00:00:00 2001 From: dragongoose Date: Sun, 16 Jul 2023 22:03:52 -0400 Subject: [PATCH 1/2] Split fetchers into multiple files for readability --- extractor/twitch/Badges.go | 66 +++ extractor/twitch/Discover.go | 130 ++++++ extractor/twitch/Search.go | 103 +++++ extractor/twitch/Stream.go | 91 ++++ extractor/twitch/Streamer.go | 169 ++++++++ extractor/twitch/VODPreview.go | 56 +++ extractor/twitch/twitchExtractor.go | 631 ---------------------------- extractor/twitch/utils.go | 58 +++ 8 files changed, 673 insertions(+), 631 deletions(-) create mode 100644 extractor/twitch/Badges.go create mode 100644 extractor/twitch/Discover.go create mode 100644 extractor/twitch/Search.go create mode 100644 extractor/twitch/Stream.go create mode 100644 extractor/twitch/Streamer.go create mode 100644 extractor/twitch/VODPreview.go delete mode 100644 extractor/twitch/twitchExtractor.go create mode 100644 extractor/twitch/utils.go diff --git a/extractor/twitch/Badges.go b/extractor/twitch/Badges.go new file mode 100644 index 0000000..8617734 --- /dev/null +++ b/extractor/twitch/Badges.go @@ -0,0 +1,66 @@ +package twitch + +import ( + "safetwitch-backend/extractor/structs" + + "github.com/tidwall/gjson" +) + +func GetTwitchBadges() ([]structs.Badge, error) { + payload := []TwitchPayload{ + { + "operationName": "ChannelPointsPredictionBadges", + "variables": map[string]interface{}{}, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "sha256Hash": "36995b30b22c31d1cd0aa329987ac9b5368bb7e6e1ab1df42808bdaa80a6dbf9", + "version": 1, + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return []structs.Badge{}, err + } + + rawBadges := gjson.Get(string(body), "0.data.badges") + formattedBadges, err := ParseBadges(rawBadges) + if err != nil { + return []structs.Badge{}, err + } + + return formattedBadges, nil +} + +func GetStreamerBadges(streamerName string) ([]structs.Badge, error) { + payload := []TwitchPayload{ + { + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "sha256Hash": "86f43113c04606e6476e39dcd432dee47c994d77a83e54b732e11d4935f0cd08", + "version": 1, + }, + }, + "operationName": "ChatList_Badges", + "variables": map[string]interface{}{ + "channelLogin": streamerName, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return []structs.Badge{}, err + } + + rawBadges := gjson.Get(string(body), "0.data.user.broadcastBadges") + + formattedBadges, err := ParseBadges(rawBadges) + if err != nil { + return []structs.Badge{}, err + } + + return formattedBadges, nil +} diff --git a/extractor/twitch/Discover.go b/extractor/twitch/Discover.go new file mode 100644 index 0000000..1db98b5 --- /dev/null +++ b/extractor/twitch/Discover.go @@ -0,0 +1,130 @@ +package twitch + +import ( + "safetwitch-backend/extractor" + "safetwitch-backend/extractor/structs" + + "github.com/tidwall/gjson" +) + +func GetDiscoveryPage(limit int, cursor string) ([]structs.CategoryPreview, error) { + payload := []TwitchPayload{ + { + "operationName": "BrowsePage_AllDirectories", + "variables": map[string]interface{}{ + "limit": limit, + "options": map[string]interface{}{ + "recommendationsContext": map[string]interface{}{ + "platform": "web", + }, + "requestID": "JIRA-VXP-2397", + "sort": "RELEVANCE", + "tags": []string{}, + }, + "cursor": cursor, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "1d1914ca3cbfaa607ecd5595b2e305e96acf987c8f25328f7713b25f604c4668", + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return []structs.CategoryPreview{}, err + } + + var parsedCategoryArray []structs.CategoryPreview + categoryArray := gjson.Get(string(body), "0.data.directoriesWithTags.edges") + for _, categoryRes := range categoryArray.Array() { + parsedCategory, err := ParseCategory(categoryRes) + if err != nil { + return []structs.CategoryPreview{}, nil + } + + parsedCategoryArray = append(parsedCategoryArray, parsedCategory) + } + + return parsedCategoryArray, nil +} + +func GetDiscoveryItem(name string, streamLimit int, cursor string) (structs.CategoryData, error) { + payload := []TwitchPayload{ + { + "operationName": "DirectoryPage_Game", + "variables": map[string]interface{}{ + "cursor": cursor, + "imageWidth": 50, + "name": name, + "options": map[string]interface{}{ + "sort": "RELEVANCE", + "recommendationsContext": map[string]interface{}{ + "platform": "web", + }, + "requestID": "JIRA-VXP-2397", + "freeformTags": nil, + "tags": []string{}, + }, + "sortTypeIsRecency": false, + "limit": streamLimit, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "df4bb6cc45055237bfaf3ead608bbafb79815c7100b6ee126719fac3762ddf8b", + }, + }, + }, + { + "operationName": "Directory_DirectoryBanner", + "variables": map[string]interface{}{ + "name": name, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "2670fbecd8fbea0211c56528d6eff5752ef9d6c73cd5238d395784b46335ded4", + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return structs.CategoryData{}, err + } + + categoryStreams := gjson.Get(string(body), "0.data.game.streams.edges") + + var parsedStreams []structs.CategoryMinifiedStream + for _, stream := range categoryStreams.Array() { + parsed, err := ParseMinifiedStream(stream) + if err != nil { + return structs.CategoryData{}, err + } + parsedStreams = append(parsedStreams, parsed) + } + + categoryData := gjson.Get(string(body), "1.data.game") + + var tags []string + for _, tag := range categoryData.Get("tags").Array() { + tags = append(tags, tag.Get("localizedName").String()) + } + + parsedCategory := structs.CategoryData{ + Name: categoryData.Get("name").String(), + DisplayName: categoryData.Get("displayName").String(), + Description: categoryData.Get("description").String(), + Viewers: int(categoryData.Get("viewersCount").Int()), + Followers: int(categoryData.Get("followersCount").Int()), + Tags: tags, + Cover: extractor.ProxyUrl(categoryData.Get("avatarURL").String()), + Streams: parsedStreams, + } + + return parsedCategory, err +} diff --git a/extractor/twitch/Search.go b/extractor/twitch/Search.go new file mode 100644 index 0000000..1b24801 --- /dev/null +++ b/extractor/twitch/Search.go @@ -0,0 +1,103 @@ +package twitch + +import ( + "safetwitch-backend/extractor" + "safetwitch-backend/extractor/structs" + + "github.com/tidwall/gjson" +) + +func GetSearchResult(query string) (structs.SearchResult, error) { + payload := []TwitchPayload{ + { + "operationName": "SearchResultsPage_SearchResults", + "variables": map[string]interface{}{ + "query": query, + "options": nil, + "requestID": "75948144-d051-4203-8511-57f3ee9b809a", + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "6ea6e6f66006485e41dbe3ebd69d5674c5b22896ce7b595d7fce6411a3790138", + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return structs.SearchResult{}, err + } + + rawStreamers := gjson.Get(string(body), "0.data.searchFor.channels.edges") + parsedStreamers := []structs.Streamer{} + for _, streamer := range rawStreamers.Array() { + stream := streamer.Get("item.stream").String() + + parsedStreamers = append(parsedStreamers, structs.Streamer{ + Username: streamer.Get("item.login").String(), + Followers: int(streamer.Get("item.followers.totalCount").Int()), + IsLive: !(stream == ""), + About: streamer.Get("item.description").String(), + Pfp: extractor.ProxyUrl(streamer.Get("item.profileImageURL").String()), + IsPartner: false, + ColorHex: nil, + Id: streamer.Get("item.channel.id").String(), + }) + } + + parsedCategories := []structs.CategoryPreview{} + rawCategories := gjson.Get(string(body), "0.data.searchFor.games.edges") + + for _, category := range rawCategories.Array() { + var tags []string + for _, tag := range category.Get("item.tags").Array() { + tags = append(tags, tag.Get("tagName").String()) + } + + parsedCategories = append(parsedCategories, structs.CategoryPreview{ + Name: category.Get("item.name").String(), + DisplayName: category.Get("item.displayName").String(), + Viewers: int(category.Get("item.viewersCount").Int()), + Image: category.Get("item.boxArtURL").String(), + Tags: tags, + }) + } + + foundRelatedLiveChannels := []structs.Streamer{} + streams := gjson.Get(string(body), "0.data.searchFor.relatedLiveChannels.edges") + for _, channel := range streams.Array() { + name := channel.Get("item.stream.broadcaster.login").String() + channel, err := GetStreamerInfo(name) + if err != nil { + return structs.SearchResult{}, nil + } + foundRelatedLiveChannels = append(foundRelatedLiveChannels, channel) + } + + foundChannelsWithTag := []structs.Streamer{} + streams = gjson.Get(string(body), "0.data.searchFor.channelsWithTag.edges") + for _, stream := range streams.Array() { + foundChannelsWithTag = append(foundChannelsWithTag, structs.Streamer{ + Username: stream.Get("item.login").String(), + Followers: int(stream.Get("item.followers.totalCount").Int()), + IsLive: !(stream.Get("stream").String() != ""), + About: stream.Get("item.description").String(), + Pfp: extractor.ProxyUrl(stream.Get("item.profileImageURL").String()), + IsPartner: false, + ColorHex: nil, + Id: stream.Get("item.channel.id").String(), + }) + } + + final := structs.SearchResult{ + Channels: parsedStreamers, + Categories: parsedCategories, + RelatedLiveChannels: foundRelatedLiveChannels, + ChannelsWithTag: foundChannelsWithTag, + } + + return final, nil + +} diff --git a/extractor/twitch/Stream.go b/extractor/twitch/Stream.go new file mode 100644 index 0000000..78dbaa6 --- /dev/null +++ b/extractor/twitch/Stream.go @@ -0,0 +1,91 @@ +package twitch + +import ( + "io" + "net/http" + "strings" + + "github.com/tidwall/gjson" +) + +func GetStream(streamerName string) (string, error) { + // STAGE 1 + // Get playback token from twitch + // Same request browser makes + payload1 := []TwitchPayload{ + { + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712", + }, + }, + "operationName": "PlaybackAccessToken", + "variables": map[string]interface{}{ + "isLive": true, + "login": streamerName, + "isVod": false, + "vodID": "", + "playerType": "site", + }, + }, + } + + _, body, err := parseResponse(payload1) + if err != nil { + return "", err + } + // These will be needed to get playlist file from twitch + token := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.value").String() + signature := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.signature").String() + if err != nil { + return "", err + } + + playlistUrl := "https://usher.ttvnw.net/api/channel/hls/" + strings.ToLower(streamerName) + ".m3u8" + params := "?sig=" + signature + "&token=" + token + + req, err := http.NewRequest("GET", playlistUrl+params, nil) + req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") + if err != nil { + return "", err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + + playlistFile, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + // holy zooks, what the scallop??? we got the playlist, houston!!! + // time to proxy all the urls!!! + proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), false) + return proxiedPlaylist, nil + +} + +func GetSubPlaylist(rawurl string) (string, error) { + req, err := http.NewRequest("GET", rawurl, nil) + + req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") + if err != nil { + return "", err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + + playlistFile, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), true) + return proxiedPlaylist, nil +} diff --git a/extractor/twitch/Streamer.go b/extractor/twitch/Streamer.go new file mode 100644 index 0000000..8f5d2e2 --- /dev/null +++ b/extractor/twitch/Streamer.go @@ -0,0 +1,169 @@ +package twitch + +import ( + "errors" + "safetwitch-backend/extractor/structs" + + "github.com/tidwall/gjson" +) + +func GetStreamerInfo(streamerName string) (structs.Streamer, error) { + payload := []TwitchPayload{ + // Streamer data + { + "operationName": "ChannelRoot_AboutPanel", + "variables": map[string]interface{}{ + "channelLogin": streamerName, + "skipSchedule": false, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6", + }, + }, + }, + + // Stream metadata + { + "operationName": "StreamMetadata", + "variables": map[string]interface{}{ + "channelLogin": streamerName, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962", + }, + }, + }, + + // Stream tags + { + "operationName": "StreamTagsTrackingChannel", + "variables": map[string]interface{}{ + "channel": streamerName, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "6aa3851aaaf88c320d514eb173563d430b28ed70fdaaf7eeef6ed4b812f48608", + }, + }, + }, + + // Stream preview image + { + "operationName": "VideoPreviewOverlay", + "variables": map[string]interface{}{ + "login": streamerName, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f", + }, + }, + }, + + // Current views + { + "operationName": "UseViewCount", + "variables": map[string]interface{}{ + "channelLogin": streamerName, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "00b11c9c428f79ae228f30080a06ffd8226a1f068d6f52fbc057cbde66e994c2", + }, + }, + }, + + // Get offline banner image + { + "operationName": "ChannelShell", + "variables": map[string]interface{}{ + "login": streamerName, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "580ab410bcd0c1ad194224957ae2241e5d252b2c5173d8e0cce9d32d5bb14efe", + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return structs.Streamer{}, err + } + + // begin parsing response + streamerFound := gjson.Get(string(body), "0.data.user") + if streamerFound.String() == "" { + return structs.Streamer{}, errors.New("streamer not found") + } + + streamerData := gjson.Get(string(body), "0.data") + parsedSocials, err := ParseSocials(streamerData.String()) + if err != nil { + return structs.Streamer{}, err + } + + parsedStream, err := ParseStream(string(body)) + var isLive bool + if err != nil { + if err.Error() == "streamer is not live" { + parsedStream = nil + } else { + return structs.Streamer{}, err + } + } + if parsedStream != nil { + isLive = true + } else { + isLive = false + } + + streamerBanner := gjson.Get(string(body), "5.data.userOrError.bannerImageURL").String() + + parsedStreamer, err := ParseStreamer(streamerData, isLive, parsedSocials, parsedStream, streamerName, streamerBanner) + if err != nil { + return structs.Streamer{}, err + } + + return parsedStreamer, nil +} + +func GetStreamerId(channelName string) (string, error) { + payload := []TwitchPayload{ + { + "operationName": "ChannelRoot_AboutPanel", + "variables": map[string]interface{}{ + "channelLogin": channelName, + "skipSchedule": true, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6", + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return "", err + } + + id := gjson.Get(string(body), "0.data.user.id").String() + + if id != "" { + return id, nil + } else { + return "", errors.New("could not find user") + } +} diff --git a/extractor/twitch/VODPreview.go b/extractor/twitch/VODPreview.go new file mode 100644 index 0000000..ad9b096 --- /dev/null +++ b/extractor/twitch/VODPreview.go @@ -0,0 +1,56 @@ +package twitch + +import ( + "safetwitch-backend/extractor/structs" + + "github.com/tidwall/gjson" +) + +/* +If you don't understand the meaning of shelve in +this context, go to an offline streamer on twitch +and look under the videos tag. Each row is considered +a shelve, and each can be expanded. This is to be used +on a streamers profile page +*/ +func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) { + payload := []TwitchPayload{ + { + "operationName": "ChannelVideoShelvesQuery", + "variables": map[string]interface{}{ + "channelLogin": channelName, + "first": 5, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "8afefb1ed16c4d8e20fa55024a7ed1727f63b6eca47d8d33a28500770bad8479", + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return []structs.Shelve{}, err + } + + /* + Twitch seperates videos by shelves. + There are 4 shells: + - Recent broadcasts + - Recent highlights and uploads + - Popular clips + - All videos + + Each one of these shelves can be expanded to get more data + */ + + shelves := gjson.Get(string(body), "0.data.user.videoShelves.edges") + parsedShelves := []structs.Shelve{} + for _, shelve := range shelves.Array() { + parsedShelves = append(parsedShelves, ParseShelve(shelve.Get("node"))) + } + + return parsedShelves, nil +} diff --git a/extractor/twitch/twitchExtractor.go b/extractor/twitch/twitchExtractor.go deleted file mode 100644 index 098c516..0000000 --- a/extractor/twitch/twitchExtractor.go +++ /dev/null @@ -1,631 +0,0 @@ -package twitch - -import ( - "bytes" - "encoding/json" - "errors" - "io" - "net/http" - "safetwitch-backend/extractor" - "safetwitch-backend/extractor/structs" - "strings" - - "github.com/tidwall/gjson" -) - -const twitchUrl = "https://gql.twitch.tv/gql" - -var language string = "en" - -// useful funcs -func SetLanguage(code string) { - language = code -} - -func call(url, method string, body io.Reader) (*http.Response, error) { - req, err := http.NewRequest(method, url, body) - if err != nil { - return req.Response, err - } - - req.Header.Add("Accept-Language", language) - req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") - resp, err := http.DefaultClient.Do(req) - return resp, err -} - -func parseResponse(payload []TwitchPayload) (response *[]structs.TwitchApiResponse, body []byte, err error) { - json_data, err := json.Marshal(payload) - if err != nil { - return nil, nil, err - } - - resp, err := call(twitchUrl, "POST", bytes.NewBuffer(json_data)) - if err != nil { - return nil, nil, err - } - defer resp.Body.Close() - - rawBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, nil, err - } - - var twitchResponse []structs.TwitchApiResponse - err = json.Unmarshal(rawBody, &twitchResponse) - if err != nil { - return nil, nil, err - } - - return &twitchResponse, rawBody, nil -} - -type TwitchPayload = map[string]interface{} - -func GetStreamerInfo(streamerName string) (structs.Streamer, error) { - payload := []TwitchPayload{ - // Streamer data - { - "operationName": "ChannelRoot_AboutPanel", - "variables": map[string]interface{}{ - "channelLogin": streamerName, - "skipSchedule": false, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6", - }, - }, - }, - - // Stream metadata - { - "operationName": "StreamMetadata", - "variables": map[string]interface{}{ - "channelLogin": streamerName, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962", - }, - }, - }, - - // Stream tags - { - "operationName": "StreamTagsTrackingChannel", - "variables": map[string]interface{}{ - "channel": streamerName, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "6aa3851aaaf88c320d514eb173563d430b28ed70fdaaf7eeef6ed4b812f48608", - }, - }, - }, - - // Stream preview image - { - "operationName": "VideoPreviewOverlay", - "variables": map[string]interface{}{ - "login": streamerName, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f", - }, - }, - }, - - // Current views - { - "operationName": "UseViewCount", - "variables": map[string]interface{}{ - "channelLogin": streamerName, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "00b11c9c428f79ae228f30080a06ffd8226a1f068d6f52fbc057cbde66e994c2", - }, - }, - }, - - // Get offline banner image - { - "operationName": "ChannelShell", - "variables": map[string]interface{}{ - "login": streamerName, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "580ab410bcd0c1ad194224957ae2241e5d252b2c5173d8e0cce9d32d5bb14efe", - }, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return structs.Streamer{}, err - } - - // begin parsing response - streamerFound := gjson.Get(string(body), "0.data.user") - if streamerFound.String() == "" { - return structs.Streamer{}, errors.New("streamer not found") - } - - streamerData := gjson.Get(string(body), "0.data") - parsedSocials, err := ParseSocials(streamerData.String()) - if err != nil { - return structs.Streamer{}, err - } - - parsedStream, err := ParseStream(string(body)) - var isLive bool - if err != nil { - if err.Error() == "streamer is not live" { - parsedStream = nil - } else { - return structs.Streamer{}, err - } - } - if parsedStream != nil { - isLive = true - } else { - isLive = false - } - - streamerBanner := gjson.Get(string(body), "5.data.userOrError.bannerImageURL").String() - - parsedStreamer, err := ParseStreamer(streamerData, isLive, parsedSocials, parsedStream, streamerName, streamerBanner) - if err != nil { - return structs.Streamer{}, err - } - - return parsedStreamer, nil -} - -func GetDiscoveryPage(limit int, cursor string) ([]structs.CategoryPreview, error) { - payload := []TwitchPayload{ - { - "operationName": "BrowsePage_AllDirectories", - "variables": map[string]interface{}{ - "limit": limit, - "options": map[string]interface{}{ - "recommendationsContext": map[string]interface{}{ - "platform": "web", - }, - "requestID": "JIRA-VXP-2397", - "sort": "RELEVANCE", - "tags": []string{}, - }, - "cursor": cursor, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "1d1914ca3cbfaa607ecd5595b2e305e96acf987c8f25328f7713b25f604c4668", - }, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return []structs.CategoryPreview{}, err - } - - var parsedCategoryArray []structs.CategoryPreview - categoryArray := gjson.Get(string(body), "0.data.directoriesWithTags.edges") - for _, categoryRes := range categoryArray.Array() { - parsedCategory, err := ParseCategory(categoryRes) - if err != nil { - return []structs.CategoryPreview{}, nil - } - - parsedCategoryArray = append(parsedCategoryArray, parsedCategory) - } - - return parsedCategoryArray, nil -} - -func GetDiscoveryItem(name string, streamLimit int, cursor string) (structs.CategoryData, error) { - payload := []TwitchPayload{ - { - "operationName": "DirectoryPage_Game", - "variables": map[string]interface{}{ - "cursor": cursor, - "imageWidth": 50, - "name": name, - "options": map[string]interface{}{ - "sort": "RELEVANCE", - "recommendationsContext": map[string]interface{}{ - "platform": "web", - }, - "requestID": "JIRA-VXP-2397", - "freeformTags": nil, - "tags": []string{}, - }, - "sortTypeIsRecency": false, - "limit": streamLimit, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "df4bb6cc45055237bfaf3ead608bbafb79815c7100b6ee126719fac3762ddf8b", - }, - }, - }, - { - "operationName": "Directory_DirectoryBanner", - "variables": map[string]interface{}{ - "name": name, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "2670fbecd8fbea0211c56528d6eff5752ef9d6c73cd5238d395784b46335ded4", - }, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return structs.CategoryData{}, err - } - - categoryStreams := gjson.Get(string(body), "0.data.game.streams.edges") - - var parsedStreams []structs.CategoryMinifiedStream - for _, stream := range categoryStreams.Array() { - parsed, err := ParseMinifiedStream(stream) - if err != nil { - return structs.CategoryData{}, err - } - parsedStreams = append(parsedStreams, parsed) - } - - categoryData := gjson.Get(string(body), "1.data.game") - - var tags []string - for _, tag := range categoryData.Get("tags").Array() { - tags = append(tags, tag.Get("localizedName").String()) - } - - parsedCategory := structs.CategoryData{ - Name: categoryData.Get("name").String(), - DisplayName: categoryData.Get("displayName").String(), - Description: categoryData.Get("description").String(), - Viewers: int(categoryData.Get("viewersCount").Int()), - Followers: int(categoryData.Get("followersCount").Int()), - Tags: tags, - Cover: extractor.ProxyUrl(categoryData.Get("avatarURL").String()), - Streams: parsedStreams, - } - - return parsedCategory, err -} - -func GetTwitchBadges() ([]structs.Badge, error) { - payload := []TwitchPayload{ - { - "operationName": "ChannelPointsPredictionBadges", - "variables": map[string]interface{}{}, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "sha256Hash": "36995b30b22c31d1cd0aa329987ac9b5368bb7e6e1ab1df42808bdaa80a6dbf9", - "version": 1, - }, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return []structs.Badge{}, err - } - - rawBadges := gjson.Get(string(body), "0.data.badges") - formattedBadges, err := ParseBadges(rawBadges) - if err != nil { - return []structs.Badge{}, err - } - - return formattedBadges, nil -} - -func GetStreamerBadges(streamerName string) ([]structs.Badge, error) { - payload := []TwitchPayload{ - { - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "sha256Hash": "86f43113c04606e6476e39dcd432dee47c994d77a83e54b732e11d4935f0cd08", - "version": 1, - }, - }, - "operationName": "ChatList_Badges", - "variables": map[string]interface{}{ - "channelLogin": streamerName, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return []structs.Badge{}, err - } - - rawBadges := gjson.Get(string(body), "0.data.user.broadcastBadges") - - formattedBadges, err := ParseBadges(rawBadges) - if err != nil { - return []structs.Badge{}, err - } - - return formattedBadges, nil -} - -func GetStream(streamerName string) (string, error) { - // STAGE 1 - // Get playback token from twitch - // Same request browser makes - payload1 := []TwitchPayload{ - { - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712", - }, - }, - "operationName": "PlaybackAccessToken", - "variables": map[string]interface{}{ - "isLive": true, - "login": streamerName, - "isVod": false, - "vodID": "", - "playerType": "site", - }, - }, - } - - _, body, err := parseResponse(payload1) - if err != nil { - return "", err - } - // These will be needed to get playlist file from twitch - token := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.value").String() - signature := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.signature").String() - if err != nil { - return "", err - } - - playlistUrl := "https://usher.ttvnw.net/api/channel/hls/" + strings.ToLower(streamerName) + ".m3u8" - params := "?sig=" + signature + "&token=" + token - - req, err := http.NewRequest("GET", playlistUrl+params, nil) - req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") - if err != nil { - return "", err - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - - playlistFile, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - - // holy zooks, what the scallop??? we got the playlist, houston!!! - // time to proxy all the urls!!! - proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), false) - return proxiedPlaylist, nil - -} - -func GetSubPlaylist(rawurl string) (string, error) { - req, err := http.NewRequest("GET", rawurl, nil) - - req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") - if err != nil { - return "", err - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - - playlistFile, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - - proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), true) - return proxiedPlaylist, nil -} - -func GetSearchResult(query string) (structs.SearchResult, error) { - payload := []TwitchPayload{ - { - "operationName": "SearchResultsPage_SearchResults", - "variables": map[string]interface{}{ - "query": query, - "options": nil, - "requestID": "75948144-d051-4203-8511-57f3ee9b809a", - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "6ea6e6f66006485e41dbe3ebd69d5674c5b22896ce7b595d7fce6411a3790138", - }, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return structs.SearchResult{}, err - } - - rawStreamers := gjson.Get(string(body), "0.data.searchFor.channels.edges") - parsedStreamers := []structs.Streamer{} - for _, streamer := range rawStreamers.Array() { - stream := streamer.Get("item.stream").String() - - parsedStreamers = append(parsedStreamers, structs.Streamer{ - Username: streamer.Get("item.login").String(), - Followers: int(streamer.Get("item.followers.totalCount").Int()), - IsLive: !(stream == ""), - About: streamer.Get("item.description").String(), - Pfp: extractor.ProxyUrl(streamer.Get("item.profileImageURL").String()), - IsPartner: false, - ColorHex: nil, - Id: streamer.Get("item.channel.id").String(), - }) - } - - parsedCategories := []structs.CategoryPreview{} - rawCategories := gjson.Get(string(body), "0.data.searchFor.games.edges") - - for _, category := range rawCategories.Array() { - var tags []string - for _, tag := range category.Get("item.tags").Array() { - tags = append(tags, tag.Get("tagName").String()) - } - - parsedCategories = append(parsedCategories, structs.CategoryPreview{ - Name: category.Get("item.name").String(), - DisplayName: category.Get("item.displayName").String(), - Viewers: int(category.Get("item.viewersCount").Int()), - Image: category.Get("item.boxArtURL").String(), - Tags: tags, - }) - } - - foundRelatedLiveChannels := []structs.Streamer{} - streams := gjson.Get(string(body), "0.data.searchFor.relatedLiveChannels.edges") - for _, channel := range streams.Array() { - name := channel.Get("item.stream.broadcaster.login").String() - channel, err := GetStreamerInfo(name) - if err != nil { - return structs.SearchResult{}, nil - } - foundRelatedLiveChannels = append(foundRelatedLiveChannels, channel) - } - - foundChannelsWithTag := []structs.Streamer{} - streams = gjson.Get(string(body), "0.data.searchFor.channelsWithTag.edges") - for _, stream := range streams.Array() { - foundChannelsWithTag = append(foundChannelsWithTag, structs.Streamer{ - Username: stream.Get("item.login").String(), - Followers: int(stream.Get("item.followers.totalCount").Int()), - IsLive: !(stream.Get("stream").String() != ""), - About: stream.Get("item.description").String(), - Pfp: extractor.ProxyUrl(stream.Get("item.profileImageURL").String()), - IsPartner: false, - ColorHex: nil, - Id: stream.Get("item.channel.id").String(), - }) - } - - final := structs.SearchResult{ - Channels: parsedStreamers, - Categories: parsedCategories, - RelatedLiveChannels: foundRelatedLiveChannels, - ChannelsWithTag: foundChannelsWithTag, - } - - return final, nil - -} - -func GetStreamerId(channelName string) (string, error) { - payload := []TwitchPayload{ - { - "operationName": "ChannelRoot_AboutPanel", - "variables": map[string]interface{}{ - "channelLogin": channelName, - "skipSchedule": true, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6", - }, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return "", err - } - - id := gjson.Get(string(body), "0.data.user.id").String() - - if id != "" { - return id, nil - } else { - return "", errors.New("could not find user") - } -} - -/* -If you don't understand the meaning of shelve in -this context, go to an offline streamer on twitch -and look under the videos tag. Each row is considered -a shelve, and each can be expanded. This is to be used -on a streamers profile page -*/ -func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) { - payload := []TwitchPayload{ - { - "operationName": "ChannelVideoShelvesQuery", - "variables": map[string]interface{}{ - "channelLogin": channelName, - "first": 5, - }, - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "8afefb1ed16c4d8e20fa55024a7ed1727f63b6eca47d8d33a28500770bad8479", - }, - }, - }, - } - - _, body, err := parseResponse(payload) - if err != nil { - return []structs.Shelve{}, err - } - - /* - Twitch seperates videos by shelves. - There are 4 shells: - - Recent broadcasts - - Recent highlights and uploads - - Popular clips - - All videos - - Each one of these shelves can be expanded to get more data - */ - - shelves := gjson.Get(string(body), "0.data.user.videoShelves.edges") - parsedShelves := []structs.Shelve{} - for _, shelve := range shelves.Array() { - parsedShelves = append(parsedShelves, ParseShelve(shelve.Get("node"))) - } - - return parsedShelves, nil -} diff --git a/extractor/twitch/utils.go b/extractor/twitch/utils.go new file mode 100644 index 0000000..4c04138 --- /dev/null +++ b/extractor/twitch/utils.go @@ -0,0 +1,58 @@ +package twitch + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "safetwitch-backend/extractor/structs" +) + +const twitchUrl = "https://gql.twitch.tv/gql" + +var Language string = "en" + +// useful funcs +func SetLanguage(code string) { + Language = code +} + +func call(url, method string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequest(method, url, body) + if err != nil { + return req.Response, err + } + + req.Header.Add("Accept-Language", Language) + req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") + resp, err := http.DefaultClient.Do(req) + return resp, err +} + +func parseResponse(payload []TwitchPayload) (response *[]structs.TwitchApiResponse, body []byte, err error) { + json_data, err := json.Marshal(payload) + if err != nil { + return nil, nil, err + } + + resp, err := call(twitchUrl, "POST", bytes.NewBuffer(json_data)) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + rawBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, err + } + + var twitchResponse []structs.TwitchApiResponse + err = json.Unmarshal(rawBody, &twitchResponse) + if err != nil { + return nil, nil, err + } + + return &twitchResponse, rawBody, nil +} + +type TwitchPayload = map[string]interface{} From 16c8402936db47c8f694bc4c3c43aca4efa8ef81 Mon Sep 17 00:00:00 2001 From: dragongoose Date: Mon, 17 Jul 2023 13:16:33 -0400 Subject: [PATCH 2/2] VOD support --- README.md | 25 +++ extractor/structs/parsed.go | 4 +- extractor/twitch/PlaybackAccessToken.go | 75 +++++++++ extractor/twitch/Stream.go | 46 ++---- extractor/twitch/VOD.go | 145 ++++++++++++++++++ extractor/twitch/VODPreview.go | 28 +++- extractor/twitch/parser.go | 52 +++---- .../twitch/{utils.go => twitchExtractor.go} | 0 routes/api/vods/homeShelves.go | 17 -- routes/api/vods/vods.go | 30 ++++ routes/proxy/proxy.go | 58 ++++++- 11 files changed, 392 insertions(+), 88 deletions(-) create mode 100644 extractor/twitch/PlaybackAccessToken.go create mode 100644 extractor/twitch/VOD.go rename extractor/twitch/{utils.go => twitchExtractor.go} (100%) delete mode 100644 routes/api/vods/homeShelves.go create mode 100644 routes/api/vods/vods.go diff --git a/README.md b/README.md index 5b935e8..d7feaf1 100644 --- a/README.md +++ b/README.md @@ -327,3 +327,28 @@ Gets a segment from one of the quality's manifest file. This is the actual video ### 200 Returns the stream segment, HLS streaming. + +### /proxy/vod/:vodID/video.m3u8 +**GET** +Gets the master manifest for a VOD +### 200 + +Returns the manifest file + +### /proxy/vod/sub/:encodedUrl/video.m3u8 +**GET** +Gets the sub manifest for a VOD +encodedUrl is the url to the sub manifiest from twitch encoded in base64 +### 200 +Returns the manifest file + +### /proxy/vod/sub/:encodedUrl/:segment +**GET** +Gets the sub manifest for a VOD +encodedUrl is the url to the sub manifiest from twitch encoded in base64 + +segment is the segment from the playlist file, etc 0.ts, 1.ts +### 200 + +Returns the manifest file + diff --git a/extractor/structs/parsed.go b/extractor/structs/parsed.go index 5d03cd9..761c7fd 100644 --- a/extractor/structs/parsed.go +++ b/extractor/structs/parsed.go @@ -105,8 +105,8 @@ type Video struct { Title string `json:"title"` PublishedAt time.Time `json:"publishedAt"` Views int `json:"views"` - Tag []string `json:"tags,omitempty"` - Streamer MinifiedStreamer `json:"streamer"` + Streamer Streamer `json:"streamer"` + Id string `json:"id"` } type Shelve struct { diff --git a/extractor/twitch/PlaybackAccessToken.go b/extractor/twitch/PlaybackAccessToken.go new file mode 100644 index 0000000..044f7a5 --- /dev/null +++ b/extractor/twitch/PlaybackAccessToken.go @@ -0,0 +1,75 @@ +package twitch + +import ( + "fmt" + + "github.com/tidwall/gjson" +) + +type PlaybackAccessToken struct { + Signature string + Token string +} + +// Assumes streamer is live +// If no vodID give "" +func getPlaybackAccessToken(streamerName string, vodID string) (PlaybackAccessToken, error) { + var isVod bool + var isLive bool + + if vodID != "" { + isVod = true + isLive = false + } else { + isVod = false + isLive = true + } + + payload1 := []TwitchPayload{ + { + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712", + }, + }, + "operationName": "PlaybackAccessToken", + "variables": map[string]interface{}{ + "isLive": isLive, + "login": streamerName, + "isVod": isVod, + "vodID": vodID, + "playerType": "site", + }, + }, + } + + _, body, err := parseResponse(payload1) + if err != nil { + return PlaybackAccessToken{}, err + } + + fmt.Println(string(body)) + + var token string + var signature string + if isLive { + token = gjson.Get(string(body), "0.data.streamPlaybackAccessToken.value").String() + signature = gjson.Get(string(body), "0.data.streamPlaybackAccessToken.signature").String() + if err != nil { + return PlaybackAccessToken{}, err + } + } else { + token = gjson.Get(string(body), "0.data.videoPlaybackAccessToken.value").String() + signature = gjson.Get(string(body), "0.data.videoPlaybackAccessToken.signature").String() + if err != nil { + return PlaybackAccessToken{}, err + } + } + + return PlaybackAccessToken{ + Signature: signature, + Token: token, + }, nil + +} diff --git a/extractor/twitch/Stream.go b/extractor/twitch/Stream.go index 78dbaa6..a47b6e3 100644 --- a/extractor/twitch/Stream.go +++ b/extractor/twitch/Stream.go @@ -1,52 +1,26 @@ package twitch import ( + "fmt" "io" "net/http" "strings" - - "github.com/tidwall/gjson" ) func GetStream(streamerName string) (string, error) { - // STAGE 1 - // Get playback token from twitch - // Same request browser makes - payload1 := []TwitchPayload{ - { - "extensions": map[string]interface{}{ - "persistedQuery": map[string]interface{}{ - "version": 1, - "sha256Hash": "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712", - }, - }, - "operationName": "PlaybackAccessToken", - "variables": map[string]interface{}{ - "isLive": true, - "login": streamerName, - "isVod": false, - "vodID": "", - "playerType": "site", - }, - }, - } - _, body, err := parseResponse(payload1) - if err != nil { - return "", err - } - // These will be needed to get playlist file from twitch - token := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.value").String() - signature := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.signature").String() - if err != nil { - return "", err - } + tokenwsig, err := getPlaybackAccessToken(streamerName, "") + token := tokenwsig.Token + signature := tokenwsig.Signature playlistUrl := "https://usher.ttvnw.net/api/channel/hls/" + strings.ToLower(streamerName) + ".m3u8" params := "?sig=" + signature + "&token=" + token req, err := http.NewRequest("GET", playlistUrl+params, nil) req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") + + fmt.Println(req.URL.String()) + if err != nil { return "", err } @@ -63,12 +37,12 @@ func GetStream(streamerName string) (string, error) { // holy zooks, what the scallop??? we got the playlist, houston!!! // time to proxy all the urls!!! - proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), false) + proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), false, false) return proxiedPlaylist, nil } -func GetSubPlaylist(rawurl string) (string, error) { +func GetSubPlaylist(rawurl string, isVOD bool) (string, error) { req, err := http.NewRequest("GET", rawurl, nil) req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") @@ -86,6 +60,6 @@ func GetSubPlaylist(rawurl string) (string, error) { return "", err } - proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), true) + proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), true, isVOD) return proxiedPlaylist, nil } diff --git a/extractor/twitch/VOD.go b/extractor/twitch/VOD.go new file mode 100644 index 0000000..76cc0bf --- /dev/null +++ b/extractor/twitch/VOD.go @@ -0,0 +1,145 @@ +package twitch + +import ( + "errors" + "fmt" + "io" + "net/http" + "safetwitch-backend/extractor" + "safetwitch-backend/extractor/structs" + + "github.com/tidwall/gjson" +) + +func GetStreamerFromVOD(vodID string) (string, error) { + payload := []TwitchPayload{ + { + "operationName": "VodChannelLoginQuery", + "variables": map[string]interface{}{ + "videoID": vodID, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "0c5feea4dad2565508828f16e53fe62614edf015159df4b3bca33423496ce78e", + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return "", err + } + + streamerName := gjson.Get(string(body), "0.data.video.owner.login").String() + if streamerName == "" { + return "", errors.New("streamernotfound") + } + + return streamerName, nil +} + +func GetVODPlaylist(vodID string) ([]byte, error) { + // get streamer + streamerName, err := GetStreamerFromVOD(vodID) + if err != nil { + return []byte{}, err + } + + // get playback token + tokens, err := getPlaybackAccessToken(streamerName, vodID) + if err != nil { + return []byte{}, err + } + + params := "?token=" + tokens.Token + "&sig=" + tokens.Signature + req, err := http.NewRequest("GET", "https://usher.ttvnw.net/vod/"+fmt.Sprint(vodID)+".m3u8"+params, nil) + if err != nil { + return []byte{}, err + } + + // headers + req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return []byte{}, err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return []byte{}, err + } + + proxiedPlaylist := ProxyPlaylistFile(string(body), false, true) + + return []byte(proxiedPlaylist), nil +} + +func GetVodMetadata(vodID string) (structs.Video, error) { + // get streamer + streamerName, err := GetStreamerFromVOD(vodID) + if err != nil { + return structs.Video{}, err + } + + payload := []TwitchPayload{ + { + "operationName": "VideoMetadata", + "variables": map[string]interface{}{ + "channelLogin": streamerName, + "videoID": vodID, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "49b5b8f268cdeb259d75b58dcb0c1a748e3b575003448a2333dc5cdafd49adad", + }, + }, + }, + { + "operationName": "ChannelRoot_AboutPanel", + "variables": map[string]interface{}{ + "channelLogin": streamerName, + "skipSchedule": false, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6", + }, + }, + }, + } + + _, body, err := parseResponse(payload) + if err != nil { + return structs.Video{}, err + } + + streamerData := gjson.Get(string(body), "1.data") + parsedSocials, err := ParseSocials(streamerData.String()) + if err != nil { + return structs.Video{}, err + } + parsedStreamer, err := ParseStreamer(streamerData, false, parsedSocials, nil, streamerName, "") + if err != nil { + return structs.Video{}, err + } + + videoData := gjson.Get(string(body), "0.data.video") + + return structs.Video{ + Title: videoData.Get("title").String(), + Preview: extractor.ProxyUrl(videoData.Get("previewThumbnailURL").String()), + Duration: int(videoData.Get("lengthSeconds").Int()), + Views: int(videoData.Get("viewCount").Int()), + PublishedAt: videoData.Get("publishedAt").Time(), + Game: ParseMinifiedCategory(videoData.Get("game")), + Streamer: parsedStreamer, + Id: vodID, + }, nil + +} diff --git a/extractor/twitch/VODPreview.go b/extractor/twitch/VODPreview.go index ad9b096..f1ddcce 100644 --- a/extractor/twitch/VODPreview.go +++ b/extractor/twitch/VODPreview.go @@ -13,12 +13,12 @@ and look under the videos tag. Each row is considered a shelve, and each can be expanded. This is to be used on a streamers profile page */ -func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) { +func GetStreamerVideoShelves(streamerName string) ([]structs.Shelve, error) { payload := []TwitchPayload{ { "operationName": "ChannelVideoShelvesQuery", "variables": map[string]interface{}{ - "channelLogin": channelName, + "channelLogin": streamerName, "first": 5, }, "extensions": map[string]interface{}{ @@ -28,6 +28,19 @@ func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) { }, }, }, + { + "operationName": "ChannelRoot_AboutPanel", + "variables": map[string]interface{}{ + "channelLogin": streamerName, + "skipSchedule": false, + }, + "extensions": map[string]interface{}{ + "persistedQuery": map[string]interface{}{ + "version": 1, + "sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6", + }, + }, + }, } _, body, err := parseResponse(payload) @@ -35,6 +48,15 @@ func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) { return []structs.Shelve{}, err } + streamerData := gjson.Get(string(body), "1.data") + parsedSocials, err := ParseSocials(streamerData.String()) + if err != nil { + return []structs.Shelve{}, err + } + parsedStreamer, err := ParseStreamer(streamerData, false, parsedSocials, nil, streamerName, "") + if err != nil { + return []structs.Shelve{}, err + } /* Twitch seperates videos by shelves. There are 4 shells: @@ -49,7 +71,7 @@ func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) { shelves := gjson.Get(string(body), "0.data.user.videoShelves.edges") parsedShelves := []structs.Shelve{} for _, shelve := range shelves.Array() { - parsedShelves = append(parsedShelves, ParseShelve(shelve.Get("node"))) + parsedShelves = append(parsedShelves, ParseShelve(shelve.Get("node"), parsedStreamer)) } return parsedShelves, nil diff --git a/extractor/twitch/parser.go b/extractor/twitch/parser.go index 8ef333a..53f23eb 100644 --- a/extractor/twitch/parser.go +++ b/extractor/twitch/parser.go @@ -174,11 +174,15 @@ func ParseBadges(data gjson.Result) ([]structs.Badge, error) { return formattedBadges, nil } -func StreamSubProxyUrl(url string) string { +func StreamSubProxyUrl(url string, isVOD bool) string { encodedUrl := base64.StdEncoding.EncodeToString([]byte(url)) backendUrl := os.Getenv("URL") - return backendUrl + "/proxy/stream/sub/" + encodedUrl + if isVOD { + return backendUrl + "/proxy/vod/sub/" + encodedUrl + "/video.m3u8" + } else { + return backendUrl + "/proxy/stream/sub/" + encodedUrl + } } func StreamSegmentProxyUrl(url string) string { @@ -188,7 +192,7 @@ func StreamSegmentProxyUrl(url string) string { return backendUrl + "/proxy/stream/segment/" + encodedUrl } -func ProxyPlaylistFile(playlist string, isSubPlaylist bool) string { +func ProxyPlaylistFile(playlist string, isSubPlaylist bool, isVod bool) string { // Split the playlist into individual entries entries := strings.Split(playlist, "\n")[1:] // Ignore the first line which contains the M3U header @@ -199,17 +203,25 @@ func ProxyPlaylistFile(playlist string, isSubPlaylist bool) string { if isSubPlaylist { newURL = StreamSegmentProxyUrl(entry) } else { - newURL = StreamSubProxyUrl(entry) + newURL = StreamSubProxyUrl(entry, isVod) } entries[i] = newURL } } + if isVod { + // Loop through each entry and replace the URL + for i, entry := range entries { + if strings.HasSuffix(entry, ".ts") { + entries[i] = entry + } + } + } + // Join the modified entries back into a single string separated by a newline modifiedPlaylist := "#EXTM3U\n" + strings.Join(entries, "\n") return modifiedPlaylist - } func ParseMinifiedCategory(data gjson.Result) structs.MinifiedCategory { @@ -230,13 +242,7 @@ func ParseMinifiedStreamer(data gjson.Result) structs.MinifiedStreamer { } } -func ParseVideo(data gjson.Result) structs.Video { - tags := []string{} - - for _, tag := range data.Get("contentTags").Array() { - tags = append(tags, tag.Get("localizedName").String()) - } - +func ParseVideo(data gjson.Result, streamer structs.Streamer) structs.Video { return structs.Video{ Preview: extractor.ProxyUrl(data.Get("previewThumbnailURL").String()), Game: ParseMinifiedCategory(data.Get("game")), @@ -244,18 +250,12 @@ func ParseVideo(data gjson.Result) structs.Video { Title: data.Get("title").String(), PublishedAt: data.Get("createdAt").Time(), Views: int(data.Get("viewCount").Int()), - Tag: tags, - Streamer: ParseMinifiedStreamer(data.Get("owner")), + Streamer: streamer, + Id: data.Get("id").String(), } } -func ParseClip(data gjson.Result) structs.Video { - tags := []string{} - - for _, tag := range data.Get("contentTags").Array() { - tags = append(tags, tag.Get("localizedName").String()) - } - +func ParseClip(data gjson.Result, streamer structs.Streamer) structs.Video { return structs.Video{ Preview: extractor.ProxyUrl(data.Get("thumbnailURL").String()), Game: ParseMinifiedCategory(data.Get("clipGame")), @@ -263,20 +263,20 @@ func ParseClip(data gjson.Result) structs.Video { Title: data.Get("clipTitle").String(), PublishedAt: data.Get("publishedAt").Time(), Views: int(data.Get("clipViewCount").Int()), - Tag: tags, - Streamer: ParseMinifiedStreamer(data.Get("broadcaster")), + Streamer: streamer, + Id: data.Get("id").String(), } } -func ParseShelve(data gjson.Result) structs.Shelve { +func ParseShelve(data gjson.Result, streamer structs.Streamer) structs.Shelve { rawVideos := data.Get("items").Array() parsedVideos := []structs.Video{} isClip := data.Get("type").String() == "TOP_CLIPS" for _, video := range rawVideos { if isClip { - parsedVideos = append(parsedVideos, ParseClip(video)) + parsedVideos = append(parsedVideos, ParseClip(video, streamer)) } else { - parsedVideos = append(parsedVideos, ParseVideo(video)) + parsedVideos = append(parsedVideos, ParseVideo(video, streamer)) } } diff --git a/extractor/twitch/utils.go b/extractor/twitch/twitchExtractor.go similarity index 100% rename from extractor/twitch/utils.go rename to extractor/twitch/twitchExtractor.go diff --git a/routes/api/vods/homeShelves.go b/routes/api/vods/homeShelves.go deleted file mode 100644 index f2e0ad6..0000000 --- a/routes/api/vods/homeShelves.go +++ /dev/null @@ -1,17 +0,0 @@ -package vods - -import ( - "safetwitch-backend/extractor" - "safetwitch-backend/extractor/twitch" - - "github.com/gin-gonic/gin" -) - -func Routes(route *gin.Engine) { - auth := route.Group("/api/vods") - - auth.GET("/shelve/:streamerName", func(context *gin.Context) { - data, _ := twitch.GetStreamerVideoShelves(context.Param("streamerName")) - context.JSON(200, extractor.FormatMessage(data, true)) - }) -} diff --git a/routes/api/vods/vods.go b/routes/api/vods/vods.go new file mode 100644 index 0000000..dfb380a --- /dev/null +++ b/routes/api/vods/vods.go @@ -0,0 +1,30 @@ +package vods + +import ( + "safetwitch-backend/extractor" + "safetwitch-backend/extractor/twitch" + + "github.com/gin-gonic/gin" +) + +func Routes(route *gin.Engine) { + auth := route.Group("/api/vods") + + auth.GET("/shelve/:streamerName", func(context *gin.Context) { + data, err := twitch.GetStreamerVideoShelves(context.Param("streamerName")) + if err != nil { + context.Error(err) + return + } + context.JSON(200, extractor.FormatMessage(data, true)) + }) + + auth.GET("/:streamerName", func(context *gin.Context) { + data, err := twitch.GetVodMetadata(context.Param("streamerName")) + if err != nil { + context.Error(err) + return + } + context.JSON(200, extractor.FormatMessage(data, true)) + }) +} diff --git a/routes/proxy/proxy.go b/routes/proxy/proxy.go index d35361d..5bde9c5 100644 --- a/routes/proxy/proxy.go +++ b/routes/proxy/proxy.go @@ -5,6 +5,7 @@ import ( "io" "net/http" "safetwitch-backend/extractor/twitch" + "strings" "github.com/gin-gonic/gin" ) @@ -40,7 +41,7 @@ func Routes(route *gin.Engine) { context.Error(err) } - context.Data(200, "application/text", []byte(playlistFile)) + context.Data(200, "application/vnd.apple.mpegurl", []byte(playlistFile)) }) auth.GET("/stream/sub/:encodedUrl", func(context *gin.Context) { @@ -49,12 +50,12 @@ func Routes(route *gin.Engine) { context.Error(err) } - playlistFile, err := twitch.GetSubPlaylist(string(decodedUrl)) + playlistFile, err := twitch.GetSubPlaylist(string(decodedUrl), false) if err != nil { context.Error(err) } - context.Data(200, "application/text", []byte(playlistFile)) + context.Data(200, "application/vnd.apple.mpegurl", []byte(playlistFile)) }) auth.GET("/stream/segment/:encodedUrl", func(context *gin.Context) { @@ -74,6 +75,55 @@ func Routes(route *gin.Engine) { context.Error(err) } - context.Data(200, "fuck", segment) + context.Data(200, "application/text", segment) + }) + + // vod + auth.GET("/vod/:vodID/video.m3u8", func(context *gin.Context) { + vodID := context.Param("vodID") + data, err := twitch.GetVODPlaylist(vodID) + if err != nil { + context.Error(err) + return + } + context.Data(200, "application/vnd.apple.mpegurl", data) + }) + + auth.GET("/vod/sub/:encodedUrl/video.m3u8", func(context *gin.Context) { + decodedUrl, err := b64.StdEncoding.DecodeString(context.Param("encodedUrl")) + if err != nil { + context.Error(err) + } + + playlistFile, err := twitch.GetSubPlaylist(string(decodedUrl), true) + if err != nil { + context.Error(err) + } + + context.Data(200, "application/vnd.apple.mpegurl", []byte(playlistFile)) + }) + + auth.GET("/vod/sub/:encodedUrl/:segment", func(context *gin.Context) { + decodedUrl, err := b64.StdEncoding.DecodeString(context.Param("encodedUrl")) + if err != nil { + context.Error(err) + } + + // remove the last path of url and replace with segment + tempurl := strings.Split(string(decodedUrl), "/") + newurl := strings.Join(tempurl[:len(tempurl)-1], "/") + "/" + context.Param("segment") + + segmentData, err := http.Get(string(newurl)) + if err != nil { + context.Error(err) + return + } + + segment, err := io.ReadAll(segmentData.Body) + if err != nil { + context.Error(err) + } + + context.Data(200, "application/text", segment) }) }