Merge branch 'master' into master

README.md

@@ -327,3 +327,28 @@ Gets a segment from one of the quality's manifest file. This is the actual video
 ### 200
 
 Returns the stream segment, HLS streaming.
+
+### /proxy/vod/:vodID/video.m3u8
+**GET**  
+Gets the master manifest for a VOD
+### 200
+
+Returns the manifest file
+
+### /proxy/vod/sub/:encodedUrl/video.m3u8
+**GET**  
+Gets the sub manifest for a VOD  
+encodedUrl is the url to the sub manifiest from twitch encoded in base64
+### 200
+Returns the manifest file
+
+### /proxy/vod/sub/:encodedUrl/:segment
+**GET**  
+Gets the sub manifest for a VOD  
+encodedUrl is the url to the sub manifiest from twitch encoded in base64  
+
+segment is the segment from the playlist file, etc 0.ts, 1.ts
+### 200
+
+Returns the manifest file
+

extractor/structs/parsed.go

@@ -105,8 +105,8 @@ type Video struct {
 	Title       string           `json:"title"`
 	PublishedAt time.Time        `json:"publishedAt"`
 	Views       int              `json:"views"`
-	Tag         []string         `json:"tags,omitempty"`
-	Streamer    MinifiedStreamer `json:"streamer"`
+	Streamer    Streamer         `json:"streamer"`
+	Id          string           `json:"id"`
 }
 
 type Shelve struct {

extractor/twitch/Badges.go

@@ -0,0 +1,66 @@
+package twitch
+
+import (
+	"safetwitch-backend/extractor/structs"
+
+	"github.com/tidwall/gjson"
+)
+
+func GetTwitchBadges() ([]structs.Badge, error) {
+	payload := []TwitchPayload{
+		{
+			"operationName": "ChannelPointsPredictionBadges",
+			"variables":     map[string]interface{}{},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"sha256Hash": "36995b30b22c31d1cd0aa329987ac9b5368bb7e6e1ab1df42808bdaa80a6dbf9",
+					"version":    1,
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return []structs.Badge{}, err
+	}
+
+	rawBadges := gjson.Get(string(body), "0.data.badges")
+	formattedBadges, err := ParseBadges(rawBadges)
+	if err != nil {
+		return []structs.Badge{}, err
+	}
+
+	return formattedBadges, nil
+}
+
+func GetStreamerBadges(streamerName string) ([]structs.Badge, error) {
+	payload := []TwitchPayload{
+		{
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"sha256Hash": "86f43113c04606e6476e39dcd432dee47c994d77a83e54b732e11d4935f0cd08",
+					"version":    1,
+				},
+			},
+			"operationName": "ChatList_Badges",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return []structs.Badge{}, err
+	}
+
+	rawBadges := gjson.Get(string(body), "0.data.user.broadcastBadges")
+
+	formattedBadges, err := ParseBadges(rawBadges)
+	if err != nil {
+		return []structs.Badge{}, err
+	}
+
+	return formattedBadges, nil
+}

extractor/twitch/Discover.go

@@ -0,0 +1,130 @@
+package twitch
+
+import (
+	"safetwitch-backend/extractor"
+	"safetwitch-backend/extractor/structs"
+
+	"github.com/tidwall/gjson"
+)
+
+func GetDiscoveryPage(limit int, cursor string) ([]structs.CategoryPreview, error) {
+	payload := []TwitchPayload{
+		{
+			"operationName": "BrowsePage_AllDirectories",
+			"variables": map[string]interface{}{
+				"limit": limit,
+				"options": map[string]interface{}{
+					"recommendationsContext": map[string]interface{}{
+						"platform": "web",
+					},
+					"requestID": "JIRA-VXP-2397",
+					"sort":      "RELEVANCE",
+					"tags":      []string{},
+				},
+				"cursor": cursor,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "1d1914ca3cbfaa607ecd5595b2e305e96acf987c8f25328f7713b25f604c4668",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return []structs.CategoryPreview{}, err
+	}
+
+	var parsedCategoryArray []structs.CategoryPreview
+	categoryArray := gjson.Get(string(body), "0.data.directoriesWithTags.edges")
+	for _, categoryRes := range categoryArray.Array() {
+		parsedCategory, err := ParseCategory(categoryRes)
+		if err != nil {
+			return []structs.CategoryPreview{}, nil
+		}
+
+		parsedCategoryArray = append(parsedCategoryArray, parsedCategory)
+	}
+
+	return parsedCategoryArray, nil
+}
+
+func GetDiscoveryItem(name string, streamLimit int, cursor string) (structs.CategoryData, error) {
+	payload := []TwitchPayload{
+		{
+			"operationName": "DirectoryPage_Game",
+			"variables": map[string]interface{}{
+				"cursor":     cursor,
+				"imageWidth": 50,
+				"name":       name,
+				"options": map[string]interface{}{
+					"sort": "RELEVANCE",
+					"recommendationsContext": map[string]interface{}{
+						"platform": "web",
+					},
+					"requestID":    "JIRA-VXP-2397",
+					"freeformTags": nil,
+					"tags":         []string{},
+				},
+				"sortTypeIsRecency": false,
+				"limit":             streamLimit,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "df4bb6cc45055237bfaf3ead608bbafb79815c7100b6ee126719fac3762ddf8b",
+				},
+			},
+		},
+		{
+			"operationName": "Directory_DirectoryBanner",
+			"variables": map[string]interface{}{
+				"name": name,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "2670fbecd8fbea0211c56528d6eff5752ef9d6c73cd5238d395784b46335ded4",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return structs.CategoryData{}, err
+	}
+
+	categoryStreams := gjson.Get(string(body), "0.data.game.streams.edges")
+
+	var parsedStreams []structs.CategoryMinifiedStream
+	for _, stream := range categoryStreams.Array() {
+		parsed, err := ParseMinifiedStream(stream)
+		if err != nil {
+			return structs.CategoryData{}, err
+		}
+		parsedStreams = append(parsedStreams, parsed)
+	}
+
+	categoryData := gjson.Get(string(body), "1.data.game")
+
+	var tags []string
+	for _, tag := range categoryData.Get("tags").Array() {
+		tags = append(tags, tag.Get("localizedName").String())
+	}
+
+	parsedCategory := structs.CategoryData{
+		Name:        categoryData.Get("name").String(),
+		DisplayName: categoryData.Get("displayName").String(),
+		Description: categoryData.Get("description").String(),
+		Viewers:     int(categoryData.Get("viewersCount").Int()),
+		Followers:   int(categoryData.Get("followersCount").Int()),
+		Tags:        tags,
+		Cover:       extractor.ProxyUrl(categoryData.Get("avatarURL").String()),
+		Streams:     parsedStreams,
+	}
+
+	return parsedCategory, err
+}

extractor/twitch/PlaybackAccessToken.go

@@ -0,0 +1,75 @@
+package twitch
+
+import (
+	"fmt"
+
+	"github.com/tidwall/gjson"
+)
+
+type PlaybackAccessToken struct {
+	Signature string
+	Token     string
+}
+
+// Assumes streamer is live
+// If no vodID give ""
+func getPlaybackAccessToken(streamerName string, vodID string) (PlaybackAccessToken, error) {
+	var isVod bool
+	var isLive bool
+
+	if vodID != "" {
+		isVod = true
+		isLive = false
+	} else {
+		isVod = false
+		isLive = true
+	}
+
+	payload1 := []TwitchPayload{
+		{
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712",
+				},
+			},
+			"operationName": "PlaybackAccessToken",
+			"variables": map[string]interface{}{
+				"isLive":     isLive,
+				"login":      streamerName,
+				"isVod":      isVod,
+				"vodID":      vodID,
+				"playerType": "site",
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload1)
+	if err != nil {
+		return PlaybackAccessToken{}, err
+	}
+
+	fmt.Println(string(body))
+
+	var token string
+	var signature string
+	if isLive {
+		token = gjson.Get(string(body), "0.data.streamPlaybackAccessToken.value").String()
+		signature = gjson.Get(string(body), "0.data.streamPlaybackAccessToken.signature").String()
+		if err != nil {
+			return PlaybackAccessToken{}, err
+		}
+	} else {
+		token = gjson.Get(string(body), "0.data.videoPlaybackAccessToken.value").String()
+		signature = gjson.Get(string(body), "0.data.videoPlaybackAccessToken.signature").String()
+		if err != nil {
+			return PlaybackAccessToken{}, err
+		}
+	}
+
+	return PlaybackAccessToken{
+		Signature: signature,
+		Token:     token,
+	}, nil
+
+}

extractor/twitch/Search.go

@@ -0,0 +1,103 @@
+package twitch
+
+import (
+	"safetwitch-backend/extractor"
+	"safetwitch-backend/extractor/structs"
+
+	"github.com/tidwall/gjson"
+)
+
+func GetSearchResult(query string) (structs.SearchResult, error) {
+	payload := []TwitchPayload{
+		{
+			"operationName": "SearchResultsPage_SearchResults",
+			"variables": map[string]interface{}{
+				"query":     query,
+				"options":   nil,
+				"requestID": "75948144-d051-4203-8511-57f3ee9b809a",
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "6ea6e6f66006485e41dbe3ebd69d5674c5b22896ce7b595d7fce6411a3790138",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return structs.SearchResult{}, err
+	}
+
+	rawStreamers := gjson.Get(string(body), "0.data.searchFor.channels.edges")
+	parsedStreamers := []structs.Streamer{}
+	for _, streamer := range rawStreamers.Array() {
+		stream := streamer.Get("item.stream").String()
+
+		parsedStreamers = append(parsedStreamers, structs.Streamer{
+			Username:  streamer.Get("item.login").String(),
+			Followers: int(streamer.Get("item.followers.totalCount").Int()),
+			IsLive:    !(stream == ""),
+			About:     streamer.Get("item.description").String(),
+			Pfp:       extractor.ProxyUrl(streamer.Get("item.profileImageURL").String()),
+			IsPartner: false,
+			ColorHex:  nil,
+			Id:        streamer.Get("item.channel.id").String(),
+		})
+	}
+
+	parsedCategories := []structs.CategoryPreview{}
+	rawCategories := gjson.Get(string(body), "0.data.searchFor.games.edges")
+
+	for _, category := range rawCategories.Array() {
+		var tags []string
+		for _, tag := range category.Get("item.tags").Array() {
+			tags = append(tags, tag.Get("tagName").String())
+		}
+
+		parsedCategories = append(parsedCategories, structs.CategoryPreview{
+			Name:        category.Get("item.name").String(),
+			DisplayName: category.Get("item.displayName").String(),
+			Viewers:     int(category.Get("item.viewersCount").Int()),
+			Image:       category.Get("item.boxArtURL").String(),
+			Tags:        tags,
+		})
+	}
+
+	foundRelatedLiveChannels := []structs.Streamer{}
+	streams := gjson.Get(string(body), "0.data.searchFor.relatedLiveChannels.edges")
+	for _, channel := range streams.Array() {
+		name := channel.Get("item.stream.broadcaster.login").String()
+		channel, err := GetStreamerInfo(name)
+		if err != nil {
+			return structs.SearchResult{}, nil
+		}
+		foundRelatedLiveChannels = append(foundRelatedLiveChannels, channel)
+	}
+
+	foundChannelsWithTag := []structs.Streamer{}
+	streams = gjson.Get(string(body), "0.data.searchFor.channelsWithTag.edges")
+	for _, stream := range streams.Array() {
+		foundChannelsWithTag = append(foundChannelsWithTag, structs.Streamer{
+			Username:  stream.Get("item.login").String(),
+			Followers: int(stream.Get("item.followers.totalCount").Int()),
+			IsLive:    !(stream.Get("stream").String() != ""),
+			About:     stream.Get("item.description").String(),
+			Pfp:       extractor.ProxyUrl(stream.Get("item.profileImageURL").String()),
+			IsPartner: false,
+			ColorHex:  nil,
+			Id:        stream.Get("item.channel.id").String(),
+		})
+	}
+
+	final := structs.SearchResult{
+		Channels:            parsedStreamers,
+		Categories:          parsedCategories,
+		RelatedLiveChannels: foundRelatedLiveChannels,
+		ChannelsWithTag:     foundChannelsWithTag,
+	}
+
+	return final, nil
+
+}

extractor/twitch/Stream.go

@@ -0,0 +1,65 @@
+package twitch
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+)
+
+func GetStream(streamerName string) (string, error) {
+
+	tokenwsig, err := getPlaybackAccessToken(streamerName, "")
+	token := tokenwsig.Token
+	signature := tokenwsig.Signature
+
+	playlistUrl := "https://usher.ttvnw.net/api/channel/hls/" + strings.ToLower(streamerName) + ".m3u8"
+	params := "?sig=" + signature + "&token=" + token
+
+	req, err := http.NewRequest("GET", playlistUrl+params, nil)
+	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
+
+	fmt.Println(req.URL.String())
+
+	if err != nil {
+		return "", err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+
+	playlistFile, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	// holy zooks, what the scallop??? we got the playlist, houston!!!
+	// time to proxy all the urls!!!
+	proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), false, false)
+	return proxiedPlaylist, nil
+
+}
+
+func GetSubPlaylist(rawurl string, isVOD bool) (string, error) {
+	req, err := http.NewRequest("GET", rawurl, nil)
+
+	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
+	if err != nil {
+		return "", err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+
+	playlistFile, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), true, isVOD)
+	return proxiedPlaylist, nil
+}

extractor/twitch/Streamer.go

@@ -0,0 +1,169 @@
+package twitch
+
+import (
+	"errors"
+	"safetwitch-backend/extractor/structs"
+
+	"github.com/tidwall/gjson"
+)
+
+func GetStreamerInfo(streamerName string) (structs.Streamer, error) {
+	payload := []TwitchPayload{
+		// Streamer data
+		{
+			"operationName": "ChannelRoot_AboutPanel",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+				"skipSchedule": false,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
+				},
+			},
+		},
+
+		// Stream metadata
+		{
+			"operationName": "StreamMetadata",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962",
+				},
+			},
+		},
+
+		// Stream tags
+		{
+			"operationName": "StreamTagsTrackingChannel",
+			"variables": map[string]interface{}{
+				"channel": streamerName,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "6aa3851aaaf88c320d514eb173563d430b28ed70fdaaf7eeef6ed4b812f48608",
+				},
+			},
+		},
+
+		// Stream preview image
+		{
+			"operationName": "VideoPreviewOverlay",
+			"variables": map[string]interface{}{
+				"login": streamerName,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f",
+				},
+			},
+		},
+
+		// Current views
+		{
+			"operationName": "UseViewCount",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "00b11c9c428f79ae228f30080a06ffd8226a1f068d6f52fbc057cbde66e994c2",
+				},
+			},
+		},
+
+		// Get offline banner image
+		{
+			"operationName": "ChannelShell",
+			"variables": map[string]interface{}{
+				"login": streamerName,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "580ab410bcd0c1ad194224957ae2241e5d252b2c5173d8e0cce9d32d5bb14efe",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return structs.Streamer{}, err
+	}
+
+	// begin parsing response
+	streamerFound := gjson.Get(string(body), "0.data.user")
+	if streamerFound.String() == "" {
+		return structs.Streamer{}, errors.New("streamer not found")
+	}
+
+	streamerData := gjson.Get(string(body), "0.data")
+	parsedSocials, err := ParseSocials(streamerData.String())
+	if err != nil {
+		return structs.Streamer{}, err
+	}
+
+	parsedStream, err := ParseStream(string(body))
+	var isLive bool
+	if err != nil {
+		if err.Error() == "streamer is not live" {
+			parsedStream = nil
+		} else {
+			return structs.Streamer{}, err
+		}
+	}
+	if parsedStream != nil {
+		isLive = true
+	} else {
+		isLive = false
+	}
+
+	streamerBanner := gjson.Get(string(body), "5.data.userOrError.bannerImageURL").String()
+
+	parsedStreamer, err := ParseStreamer(streamerData, isLive, parsedSocials, parsedStream, streamerName, streamerBanner)
+	if err != nil {
+		return structs.Streamer{}, err
+	}
+
+	return parsedStreamer, nil
+}
+
+func GetStreamerId(channelName string) (string, error) {
+	payload := []TwitchPayload{
+		{
+			"operationName": "ChannelRoot_AboutPanel",
+			"variables": map[string]interface{}{
+				"channelLogin": channelName,
+				"skipSchedule": true,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return "", err
+	}
+
+	id := gjson.Get(string(body), "0.data.user.id").String()
+
+	if id != "" {
+		return id, nil
+	} else {
+		return "", errors.New("could not find user")
+	}
+}

extractor/twitch/VOD.go

@@ -0,0 +1,145 @@
+package twitch
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"safetwitch-backend/extractor"
+	"safetwitch-backend/extractor/structs"
+
+	"github.com/tidwall/gjson"
+)
+
+func GetStreamerFromVOD(vodID string) (string, error) {
+	payload := []TwitchPayload{
+		{
+			"operationName": "VodChannelLoginQuery",
+			"variables": map[string]interface{}{
+				"videoID": vodID,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "0c5feea4dad2565508828f16e53fe62614edf015159df4b3bca33423496ce78e",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return "", err
+	}
+
+	streamerName := gjson.Get(string(body), "0.data.video.owner.login").String()
+	if streamerName == "" {
+		return "", errors.New("streamernotfound")
+	}
+
+	return streamerName, nil
+}
+
+func GetVODPlaylist(vodID string) ([]byte, error) {
+	// get streamer
+	streamerName, err := GetStreamerFromVOD(vodID)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	// get playback token
+	tokens, err := getPlaybackAccessToken(streamerName, vodID)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	params := "?token=" + tokens.Token + "&sig=" + tokens.Signature
+	req, err := http.NewRequest("GET", "https://usher.ttvnw.net/vod/"+fmt.Sprint(vodID)+".m3u8"+params, nil)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	// headers
+	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	proxiedPlaylist := ProxyPlaylistFile(string(body), false, true)
+
+	return []byte(proxiedPlaylist), nil
+}
+
+func GetVodMetadata(vodID string) (structs.Video, error) {
+	// get streamer
+	streamerName, err := GetStreamerFromVOD(vodID)
+	if err != nil {
+		return structs.Video{}, err
+	}
+
+	payload := []TwitchPayload{
+		{
+			"operationName": "VideoMetadata",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+				"videoID":      vodID,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "49b5b8f268cdeb259d75b58dcb0c1a748e3b575003448a2333dc5cdafd49adad",
+				},
+			},
+		},
+		{
+			"operationName": "ChannelRoot_AboutPanel",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+				"skipSchedule": false,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return structs.Video{}, err
+	}
+
+	streamerData := gjson.Get(string(body), "1.data")
+	parsedSocials, err := ParseSocials(streamerData.String())
+	if err != nil {
+		return structs.Video{}, err
+	}
+	parsedStreamer, err := ParseStreamer(streamerData, false, parsedSocials, nil, streamerName, "")
+	if err != nil {
+		return structs.Video{}, err
+	}
+
+	videoData := gjson.Get(string(body), "0.data.video")
+
+	return structs.Video{
+		Title:       videoData.Get("title").String(),
+		Preview:     extractor.ProxyUrl(videoData.Get("previewThumbnailURL").String()),
+		Duration:    int(videoData.Get("lengthSeconds").Int()),
+		Views:       int(videoData.Get("viewCount").Int()),
+		PublishedAt: videoData.Get("publishedAt").Time(),
+		Game:        ParseMinifiedCategory(videoData.Get("game")),
+		Streamer:    parsedStreamer,
+		Id:          vodID,
+	}, nil
+
+}

extractor/twitch/VODPreview.go

@@ -0,0 +1,78 @@
+package twitch
+
+import (
+	"safetwitch-backend/extractor/structs"
+
+	"github.com/tidwall/gjson"
+)
+
+/*
+If you don't understand the meaning of shelve in
+this context, go to an offline streamer on twitch
+and look under the videos tag. Each row is considered
+a shelve, and each can be expanded. This is to be used
+on a streamers profile page
+*/
+func GetStreamerVideoShelves(streamerName string) ([]structs.Shelve, error) {
+	payload := []TwitchPayload{
+		{
+			"operationName": "ChannelVideoShelvesQuery",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+				"first":        5,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "8afefb1ed16c4d8e20fa55024a7ed1727f63b6eca47d8d33a28500770bad8479",
+				},
+			},
+		},
+		{
+			"operationName": "ChannelRoot_AboutPanel",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+				"skipSchedule": false,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return []structs.Shelve{}, err
+	}
+
+	streamerData := gjson.Get(string(body), "1.data")
+	parsedSocials, err := ParseSocials(streamerData.String())
+	if err != nil {
+		return []structs.Shelve{}, err
+	}
+	parsedStreamer, err := ParseStreamer(streamerData, false, parsedSocials, nil, streamerName, "")
+	if err != nil {
+		return []structs.Shelve{}, err
+	}
+	/*
+		Twitch seperates videos by shelves.
+		There are 4 shells:
+		- Recent broadcasts
+		- Recent highlights and uploads
+		- Popular clips
+		- All videos
+
+		Each one of these shelves can be expanded to get more data
+	*/
+
+	shelves := gjson.Get(string(body), "0.data.user.videoShelves.edges")
+	parsedShelves := []structs.Shelve{}
+	for _, shelve := range shelves.Array() {
+		parsedShelves = append(parsedShelves, ParseShelve(shelve.Get("node"), parsedStreamer))
+	}
+
+	return parsedShelves, nil
+}

extractor/twitch/parser.go

@@ -174,12 +174,16 @@ func ParseBadges(data gjson.Result) ([]structs.Badge, error) {
 	return formattedBadges, nil
 }
 
-func StreamSubProxyUrl(url string) string {
+func StreamSubProxyUrl(url string, isVOD bool) string {
 	encodedUrl := base64.StdEncoding.EncodeToString([]byte(url))
 	backendUrl := os.Getenv("URL")
 
+	if isVOD {
+		return backendUrl + "/proxy/vod/sub/" + encodedUrl + "/video.m3u8"
+	} else {
 		return backendUrl + "/proxy/stream/sub/" + encodedUrl
 	}
+}
 
 func StreamSegmentProxyUrl(url string) string {
 	encodedUrl := base64.StdEncoding.EncodeToString([]byte(url))
@@ -188,7 +192,7 @@ func StreamSegmentProxyUrl(url string) string {
 	return backendUrl + "/proxy/stream/segment/" + encodedUrl
 }
 
-func ProxyPlaylistFile(playlist string, isSubPlaylist bool) string {
+func ProxyPlaylistFile(playlist string, isSubPlaylist bool, isVod bool) string {
 	// Split the playlist into individual entries
 	entries := strings.Split(playlist, "\n")[1:] // Ignore the first line which contains the M3U header
 
@@ -199,17 +203,25 @@ func ProxyPlaylistFile(playlist string, isSubPlaylist bool) string {
 			if isSubPlaylist {
 				newURL = StreamSegmentProxyUrl(entry)
 			} else {
-				newURL = StreamSubProxyUrl(entry)
+				newURL = StreamSubProxyUrl(entry, isVod)
 			}
 			entries[i] = newURL
 		}
 	}
 
+	if isVod {
+		// Loop through each entry and replace the URL
+		for i, entry := range entries {
+			if strings.HasSuffix(entry, ".ts") {
+				entries[i] = entry
+			}
+		}
+	}
+
 	// Join the modified entries back into a single string separated by a newline
 	modifiedPlaylist := "#EXTM3U\n" + strings.Join(entries, "\n")
 
 	return modifiedPlaylist
-
 }
 
 func ParseMinifiedCategory(data gjson.Result) structs.MinifiedCategory {
@@ -230,13 +242,7 @@ func ParseMinifiedStreamer(data gjson.Result) structs.MinifiedStreamer {
 	}
 }
 
-func ParseVideo(data gjson.Result) structs.Video {
-	tags := []string{}
-
-	for _, tag := range data.Get("contentTags").Array() {
-		tags = append(tags, tag.Get("localizedName").String())
-	}
-
+func ParseVideo(data gjson.Result, streamer structs.Streamer) structs.Video {
 	return structs.Video{
 		Preview:     extractor.ProxyUrl(data.Get("previewThumbnailURL").String()),
 		Game:        ParseMinifiedCategory(data.Get("game")),
@@ -244,18 +250,12 @@ func ParseVideo(data gjson.Result) structs.Video {
 		Title:       data.Get("title").String(),
 		PublishedAt: data.Get("createdAt").Time(),
 		Views:       int(data.Get("viewCount").Int()),
-		Tag:         tags,
-		Streamer:    ParseMinifiedStreamer(data.Get("owner")),
+		Streamer:    streamer,
+		Id:          data.Get("id").String(),
 	}
 }
 
-func ParseClip(data gjson.Result) structs.Video {
-	tags := []string{}
-
-	for _, tag := range data.Get("contentTags").Array() {
-		tags = append(tags, tag.Get("localizedName").String())
-	}
-
+func ParseClip(data gjson.Result, streamer structs.Streamer) structs.Video {
 	return structs.Video{
 		Preview:     extractor.ProxyUrl(data.Get("thumbnailURL").String()),
 		Game:        ParseMinifiedCategory(data.Get("clipGame")),
@@ -263,20 +263,20 @@ func ParseClip(data gjson.Result) structs.Video {
 		Title:       data.Get("clipTitle").String(),
 		PublishedAt: data.Get("publishedAt").Time(),
 		Views:       int(data.Get("clipViewCount").Int()),
-		Tag:         tags,
-		Streamer:    ParseMinifiedStreamer(data.Get("broadcaster")),
+		Streamer:    streamer,
+		Id:          data.Get("id").String(),
 	}
 }
-func ParseShelve(data gjson.Result) structs.Shelve {
+func ParseShelve(data gjson.Result, streamer structs.Streamer) structs.Shelve {
 	rawVideos := data.Get("items").Array()
 	parsedVideos := []structs.Video{}
 
 	isClip := data.Get("type").String() == "TOP_CLIPS"
 	for _, video := range rawVideos {
 		if isClip {
-			parsedVideos = append(parsedVideos, ParseClip(video))
+			parsedVideos = append(parsedVideos, ParseClip(video, streamer))
 		} else {
-			parsedVideos = append(parsedVideos, ParseVideo(video))
+			parsedVideos = append(parsedVideos, ParseVideo(video, streamer))
 		}
 	}
 

extractor/twitch/twitchExtractor.go

@@ -3,23 +3,18 @@ package twitch
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"io"
 	"net/http"
-	"safetwitch-backend/extractor"
 	"safetwitch-backend/extractor/structs"
-	"strings"
-
-	"github.com/tidwall/gjson"
 )
 
 const twitchUrl = "https://gql.twitch.tv/gql"
 
-var language string = "en"
+var Language string = "en"
 
 // useful funcs
 func SetLanguage(code string) {
-	language = code
+	Language = code
 }
 
 func call(url, method string, body io.Reader) (*http.Response, error) {
@@ -28,7 +23,7 @@ func call(url, method string, body io.Reader) (*http.Response, error) {
 		return req.Response, err
 	}
 
-	req.Header.Add("Accept-Language", language)
+	req.Header.Add("Accept-Language", Language)
 	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
 	resp, err := http.DefaultClient.Do(req)
 	return resp, err
@@ -61,571 +56,3 @@ func parseResponse(payload []TwitchPayload) (response *[]structs.TwitchApiRespon
 }
 
 type TwitchPayload = map[string]interface{}
-
-func GetStreamerInfo(streamerName string) (structs.Streamer, error) {
-	payload := []TwitchPayload{
-		// Streamer data
-		{
-			"operationName": "ChannelRoot_AboutPanel",
-			"variables": map[string]interface{}{
-				"channelLogin": streamerName,
-				"skipSchedule": false,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
-				},
-			},
-		},
-
-		// Stream metadata
-		{
-			"operationName": "StreamMetadata",
-			"variables": map[string]interface{}{
-				"channelLogin": streamerName,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962",
-				},
-			},
-		},
-
-		// Stream tags
-		{
-			"operationName": "StreamTagsTrackingChannel",
-			"variables": map[string]interface{}{
-				"channel": streamerName,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "6aa3851aaaf88c320d514eb173563d430b28ed70fdaaf7eeef6ed4b812f48608",
-				},
-			},
-		},
-
-		// Stream preview image
-		{
-			"operationName": "VideoPreviewOverlay",
-			"variables": map[string]interface{}{
-				"login": streamerName,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f",
-				},
-			},
-		},
-
-		// Current views
-		{
-			"operationName": "UseViewCount",
-			"variables": map[string]interface{}{
-				"channelLogin": streamerName,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "00b11c9c428f79ae228f30080a06ffd8226a1f068d6f52fbc057cbde66e994c2",
-				},
-			},
-		},
-
-		// Get offline banner image
-		{
-			"operationName": "ChannelShell",
-			"variables": map[string]interface{}{
-				"login": streamerName,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "580ab410bcd0c1ad194224957ae2241e5d252b2c5173d8e0cce9d32d5bb14efe",
-				},
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return structs.Streamer{}, err
-	}
-
-	// begin parsing response
-	streamerFound := gjson.Get(string(body), "0.data.user")
-	if streamerFound.String() == "" {
-		return structs.Streamer{}, errors.New("streamer not found")
-	}
-
-	streamerData := gjson.Get(string(body), "0.data")
-	parsedSocials, err := ParseSocials(streamerData.String())
-	if err != nil {
-		return structs.Streamer{}, err
-	}
-
-	parsedStream, err := ParseStream(string(body))
-	var isLive bool
-	if err != nil {
-		if err.Error() == "streamer is not live" {
-			parsedStream = nil
-		} else {
-			return structs.Streamer{}, err
-		}
-	}
-	if parsedStream != nil {
-		isLive = true
-	} else {
-		isLive = false
-	}
-
-	streamerBanner := gjson.Get(string(body), "5.data.userOrError.bannerImageURL").String()
-
-	parsedStreamer, err := ParseStreamer(streamerData, isLive, parsedSocials, parsedStream, streamerName, streamerBanner)
-	if err != nil {
-		return structs.Streamer{}, err
-	}
-
-	return parsedStreamer, nil
-}
-
-func GetDiscoveryPage(limit int, cursor string) ([]structs.CategoryPreview, error) {
-	payload := []TwitchPayload{
-		{
-			"operationName": "BrowsePage_AllDirectories",
-			"variables": map[string]interface{}{
-				"limit": limit,
-				"options": map[string]interface{}{
-					"recommendationsContext": map[string]interface{}{
-						"platform": "web",
-					},
-					"requestID": "JIRA-VXP-2397",
-					"sort":      "RELEVANCE",
-					"tags":      []string{},
-				},
-				"cursor": cursor,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "1d1914ca3cbfaa607ecd5595b2e305e96acf987c8f25328f7713b25f604c4668",
-				},
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return []structs.CategoryPreview{}, err
-	}
-
-	var parsedCategoryArray []structs.CategoryPreview
-	categoryArray := gjson.Get(string(body), "0.data.directoriesWithTags.edges")
-	for _, categoryRes := range categoryArray.Array() {
-		parsedCategory, err := ParseCategory(categoryRes)
-		if err != nil {
-			return []structs.CategoryPreview{}, nil
-		}
-
-		parsedCategoryArray = append(parsedCategoryArray, parsedCategory)
-	}
-
-	return parsedCategoryArray, nil
-}
-
-func GetDiscoveryItem(name string, streamLimit int, cursor string) (structs.CategoryData, error) {
-	payload := []TwitchPayload{
-		{
-			"operationName": "DirectoryPage_Game",
-			"variables": map[string]interface{}{
-				"cursor":     cursor,
-				"imageWidth": 50,
-				"name":       name,
-				"options": map[string]interface{}{
-					"sort": "RELEVANCE",
-					"recommendationsContext": map[string]interface{}{
-						"platform": "web",
-					},
-					"requestID":    "JIRA-VXP-2397",
-					"freeformTags": nil,
-					"tags":         []string{},
-				},
-				"sortTypeIsRecency": false,
-				"limit":             streamLimit,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "df4bb6cc45055237bfaf3ead608bbafb79815c7100b6ee126719fac3762ddf8b",
-				},
-			},
-		},
-		{
-			"operationName": "Directory_DirectoryBanner",
-			"variables": map[string]interface{}{
-				"name": name,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "2670fbecd8fbea0211c56528d6eff5752ef9d6c73cd5238d395784b46335ded4",
-				},
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return structs.CategoryData{}, err
-	}
-
-	categoryStreams := gjson.Get(string(body), "0.data.game.streams.edges")
-
-	var parsedStreams []structs.CategoryMinifiedStream
-	for _, stream := range categoryStreams.Array() {
-		parsed, err := ParseMinifiedStream(stream)
-		if err != nil {
-			return structs.CategoryData{}, err
-		}
-		parsedStreams = append(parsedStreams, parsed)
-	}
-
-	categoryData := gjson.Get(string(body), "1.data.game")
-
-	var tags []string
-	for _, tag := range categoryData.Get("tags").Array() {
-		tags = append(tags, tag.Get("localizedName").String())
-	}
-
-	parsedCategory := structs.CategoryData{
-		Name:        categoryData.Get("name").String(),
-		DisplayName: categoryData.Get("displayName").String(),
-		Description: categoryData.Get("description").String(),
-		Viewers:     int(categoryData.Get("viewersCount").Int()),
-		Followers:   int(categoryData.Get("followersCount").Int()),
-		Tags:        tags,
-		Cover:       extractor.ProxyUrl(categoryData.Get("avatarURL").String()),
-		Streams:     parsedStreams,
-	}
-
-	return parsedCategory, err
-}
-
-func GetTwitchBadges() ([]structs.Badge, error) {
-	payload := []TwitchPayload{
-		{
-			"operationName": "ChannelPointsPredictionBadges",
-			"variables":     map[string]interface{}{},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"sha256Hash": "36995b30b22c31d1cd0aa329987ac9b5368bb7e6e1ab1df42808bdaa80a6dbf9",
-					"version":    1,
-				},
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return []structs.Badge{}, err
-	}
-
-	rawBadges := gjson.Get(string(body), "0.data.badges")
-	formattedBadges, err := ParseBadges(rawBadges)
-	if err != nil {
-		return []structs.Badge{}, err
-	}
-
-	return formattedBadges, nil
-}
-
-func GetStreamerBadges(streamerName string) ([]structs.Badge, error) {
-	payload := []TwitchPayload{
-		{
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"sha256Hash": "86f43113c04606e6476e39dcd432dee47c994d77a83e54b732e11d4935f0cd08",
-					"version":    1,
-				},
-			},
-			"operationName": "ChatList_Badges",
-			"variables": map[string]interface{}{
-				"channelLogin": streamerName,
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return []structs.Badge{}, err
-	}
-
-	rawBadges := gjson.Get(string(body), "0.data.user.broadcastBadges")
-
-	formattedBadges, err := ParseBadges(rawBadges)
-	if err != nil {
-		return []structs.Badge{}, err
-	}
-
-	return formattedBadges, nil
-}
-
-func GetStream(streamerName string) (string, error) {
-	// STAGE 1
-	// Get playback token from twitch
-	// Same request browser makes
-	payload1 := []TwitchPayload{
-		{
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712",
-				},
-			},
-			"operationName": "PlaybackAccessToken",
-			"variables": map[string]interface{}{
-				"isLive":     true,
-				"login":      streamerName,
-				"isVod":      false,
-				"vodID":      "",
-				"playerType": "site",
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload1)
-	if err != nil {
-		return "", err
-	}
-	// These will be needed to get playlist file from twitch
-	token := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.value").String()
-	signature := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.signature").String()
-	if err != nil {
-		return "", err
-	}
-
-	playlistUrl := "https://usher.ttvnw.net/api/channel/hls/" + strings.ToLower(streamerName) + ".m3u8"
-	params := "?sig=" + signature + "&token=" + token
-
-	req, err := http.NewRequest("GET", playlistUrl+params, nil)
-	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
-	if err != nil {
-		return "", err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-
-	playlistFile, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-
-	// holy zooks, what the scallop??? we got the playlist, houston!!!
-	// time to proxy all the urls!!!
-	proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), false)
-	return proxiedPlaylist, nil
-
-}
-
-func GetSubPlaylist(rawurl string) (string, error) {
-	req, err := http.NewRequest("GET", rawurl, nil)
-
-	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
-	if err != nil {
-		return "", err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-
-	playlistFile, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-
-	proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), true)
-	return proxiedPlaylist, nil
-}
-
-func GetSearchResult(query string) (structs.SearchResult, error) {
-	payload := []TwitchPayload{
-		{
-			"operationName": "SearchResultsPage_SearchResults",
-			"variables": map[string]interface{}{
-				"query":     query,
-				"options":   nil,
-				"requestID": "75948144-d051-4203-8511-57f3ee9b809a",
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "6ea6e6f66006485e41dbe3ebd69d5674c5b22896ce7b595d7fce6411a3790138",
-				},
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return structs.SearchResult{}, err
-	}
-
-	rawStreamers := gjson.Get(string(body), "0.data.searchFor.channels.edges")
-	parsedStreamers := []structs.Streamer{}
-	for _, streamer := range rawStreamers.Array() {
-		stream := streamer.Get("item.stream").String()
-
-		parsedStreamers = append(parsedStreamers, structs.Streamer{
-			Username:  streamer.Get("item.login").String(),
-			Followers: int(streamer.Get("item.followers.totalCount").Int()),
-			IsLive:    !(stream == ""),
-			About:     streamer.Get("item.description").String(),
-			Pfp:       extractor.ProxyUrl(streamer.Get("item.profileImageURL").String()),
-			IsPartner: false,
-			ColorHex:  nil,
-			Id:        streamer.Get("item.channel.id").String(),
-		})
-	}
-
-	parsedCategories := []structs.CategoryPreview{}
-	rawCategories := gjson.Get(string(body), "0.data.searchFor.games.edges")
-
-	for _, category := range rawCategories.Array() {
-		var tags []string
-		for _, tag := range category.Get("item.tags").Array() {
-			tags = append(tags, tag.Get("tagName").String())
-		}
-
-		parsedCategories = append(parsedCategories, structs.CategoryPreview{
-			Name:        category.Get("item.name").String(),
-			DisplayName: category.Get("item.displayName").String(),
-			Viewers:     int(category.Get("item.viewersCount").Int()),
-			Image:       category.Get("item.boxArtURL").String(),
-			Tags:        tags,
-		})
-	}
-
-	foundRelatedLiveChannels := []structs.Streamer{}
-	streams := gjson.Get(string(body), "0.data.searchFor.relatedLiveChannels.edges")
-	for _, channel := range streams.Array() {
-		name := channel.Get("item.stream.broadcaster.login").String()
-		channel, err := GetStreamerInfo(name)
-		if err != nil {
-			return structs.SearchResult{}, nil
-		}
-		foundRelatedLiveChannels = append(foundRelatedLiveChannels, channel)
-	}
-
-	foundChannelsWithTag := []structs.Streamer{}
-	streams = gjson.Get(string(body), "0.data.searchFor.channelsWithTag.edges")
-	for _, stream := range streams.Array() {
-		foundChannelsWithTag = append(foundChannelsWithTag, structs.Streamer{
-			Username:  stream.Get("item.login").String(),
-			Followers: int(stream.Get("item.followers.totalCount").Int()),
-			IsLive:    !(stream.Get("stream").String() != ""),
-			About:     stream.Get("item.description").String(),
-			Pfp:       extractor.ProxyUrl(stream.Get("item.profileImageURL").String()),
-			IsPartner: false,
-			ColorHex:  nil,
-			Id:        stream.Get("item.channel.id").String(),
-		})
-	}
-
-	final := structs.SearchResult{
-		Channels:            parsedStreamers,
-		Categories:          parsedCategories,
-		RelatedLiveChannels: foundRelatedLiveChannels,
-		ChannelsWithTag:     foundChannelsWithTag,
-	}
-
-	return final, nil
-
-}
-
-func GetStreamerId(channelName string) (string, error) {
-	payload := []TwitchPayload{
-		{
-			"operationName": "ChannelRoot_AboutPanel",
-			"variables": map[string]interface{}{
-				"channelLogin": channelName,
-				"skipSchedule": true,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
-				},
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return "", err
-	}
-
-	id := gjson.Get(string(body), "0.data.user.id").String()
-
-	if id != "" {
-		return id, nil
-	} else {
-		return "", errors.New("could not find user")
-	}
-}
-
-/*
-If you don't understand the meaning of shelve in
-this context, go to an offline streamer on twitch
-and look under the videos tag. Each row is considered
-a shelve, and each can be expanded. This is to be used
-on a streamers profile page
-*/
-func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) {
-	payload := []TwitchPayload{
-		{
-			"operationName": "ChannelVideoShelvesQuery",
-			"variables": map[string]interface{}{
-				"channelLogin": channelName,
-				"first":        5,
-			},
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "8afefb1ed16c4d8e20fa55024a7ed1727f63b6eca47d8d33a28500770bad8479",
-				},
-			},
-		},
-	}
-
-	_, body, err := parseResponse(payload)
-	if err != nil {
-		return []structs.Shelve{}, err
-	}
-
-	/*
-		Twitch seperates videos by shelves.
-		There are 4 shells:
-		- Recent broadcasts
-		- Recent highlights and uploads
-		- Popular clips
-		- All videos
-
-		Each one of these shelves can be expanded to get more data
-	*/
-
-	shelves := gjson.Get(string(body), "0.data.user.videoShelves.edges")
-	parsedShelves := []structs.Shelve{}
-	for _, shelve := range shelves.Array() {
-		parsedShelves = append(parsedShelves, ParseShelve(shelve.Get("node")))
-	}
-
-	return parsedShelves, nil
-}

routes/api/vods/homeShelves.go

@@ -1,17 +0,0 @@
-package vods
-
-import (
-	"safetwitch-backend/extractor"
-	"safetwitch-backend/extractor/twitch"
-
-	"github.com/gin-gonic/gin"
-)
-
-func Routes(route *gin.Engine) {
-	auth := route.Group("/api/vods")
-
-	auth.GET("/shelve/:streamerName", func(context *gin.Context) {
-		data, _ := twitch.GetStreamerVideoShelves(context.Param("streamerName"))
-		context.JSON(200, extractor.FormatMessage(data, true))
-	})
-}

routes/api/vods/vods.go

@@ -0,0 +1,30 @@
+package vods
+
+import (
+	"safetwitch-backend/extractor"
+	"safetwitch-backend/extractor/twitch"
+
+	"github.com/gin-gonic/gin"
+)
+
+func Routes(route *gin.Engine) {
+	auth := route.Group("/api/vods")
+
+	auth.GET("/shelve/:streamerName", func(context *gin.Context) {
+		data, err := twitch.GetStreamerVideoShelves(context.Param("streamerName"))
+		if err != nil {
+			context.Error(err)
+			return
+		}
+		context.JSON(200, extractor.FormatMessage(data, true))
+	})
+
+	auth.GET("/:streamerName", func(context *gin.Context) {
+		data, err := twitch.GetVodMetadata(context.Param("streamerName"))
+		if err != nil {
+			context.Error(err)
+			return
+		}
+		context.JSON(200, extractor.FormatMessage(data, true))
+	})
+}

routes/proxy/proxy.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"net/http"
 	"safetwitch-backend/extractor/twitch"
+	"strings"
 
 	"github.com/gin-gonic/gin"
 )
@@ -40,7 +41,7 @@ func Routes(route *gin.Engine) {
 			context.Error(err)
 		}
 
-		context.Data(200, "application/text", []byte(playlistFile))
+		context.Data(200, "application/vnd.apple.mpegurl", []byte(playlistFile))
 	})
 
 	auth.GET("/stream/sub/:encodedUrl", func(context *gin.Context) {
@@ -49,12 +50,12 @@ func Routes(route *gin.Engine) {
 			context.Error(err)
 		}
 
-		playlistFile, err := twitch.GetSubPlaylist(string(decodedUrl))
+		playlistFile, err := twitch.GetSubPlaylist(string(decodedUrl), false)
 		if err != nil {
 			context.Error(err)
 		}
 
-		context.Data(200, "application/text", []byte(playlistFile))
+		context.Data(200, "application/vnd.apple.mpegurl", []byte(playlistFile))
 	})
 
 	auth.GET("/stream/segment/:encodedUrl", func(context *gin.Context) {
@@ -74,6 +75,55 @@ func Routes(route *gin.Engine) {
 			context.Error(err)
 		}
 
-		context.Data(200, "fuck", segment)
+		context.Data(200, "application/text", segment)
+	})
+
+	// vod
+	auth.GET("/vod/:vodID/video.m3u8", func(context *gin.Context) {
+		vodID := context.Param("vodID")
+		data, err := twitch.GetVODPlaylist(vodID)
+		if err != nil {
+			context.Error(err)
+			return
+		}
+		context.Data(200, "application/vnd.apple.mpegurl", data)
+	})
+
+	auth.GET("/vod/sub/:encodedUrl/video.m3u8", func(context *gin.Context) {
+		decodedUrl, err := b64.StdEncoding.DecodeString(context.Param("encodedUrl"))
+		if err != nil {
+			context.Error(err)
+		}
+
+		playlistFile, err := twitch.GetSubPlaylist(string(decodedUrl), true)
+		if err != nil {
+			context.Error(err)
+		}
+
+		context.Data(200, "application/vnd.apple.mpegurl", []byte(playlistFile))
+	})
+
+	auth.GET("/vod/sub/:encodedUrl/:segment", func(context *gin.Context) {
+		decodedUrl, err := b64.StdEncoding.DecodeString(context.Param("encodedUrl"))
+		if err != nil {
+			context.Error(err)
+		}
+
+		// remove the last path of url and replace with segment
+		tempurl := strings.Split(string(decodedUrl), "/")
+		newurl := strings.Join(tempurl[:len(tempurl)-1], "/") + "/" + context.Param("segment")
+
+		segmentData, err := http.Get(string(newurl))
+		if err != nil {
+			context.Error(err)
+			return
+		}
+
+		segment, err := io.ReadAll(segmentData.Body)
+		if err != nil {
+			context.Error(err)
+		}
+
+		context.Data(200, "application/text", segment)
 	})
 }