VOD support

README.md

@@ -327,3 +327,28 @@ Gets a segment from one of the quality's manifest file. This is the actual video
 ### 200
 
 Returns the stream segment, HLS streaming.
+
+### /proxy/vod/:vodID/video.m3u8
+**GET**  
+Gets the master manifest for a VOD
+### 200
+
+Returns the manifest file
+
+### /proxy/vod/sub/:encodedUrl/video.m3u8
+**GET**  
+Gets the sub manifest for a VOD  
+encodedUrl is the url to the sub manifiest from twitch encoded in base64
+### 200
+Returns the manifest file
+
+### /proxy/vod/sub/:encodedUrl/:segment
+**GET**  
+Gets the sub manifest for a VOD  
+encodedUrl is the url to the sub manifiest from twitch encoded in base64  
+
+segment is the segment from the playlist file, etc 0.ts, 1.ts
+### 200
+
+Returns the manifest file
+

extractor/structs/parsed.go

@@ -105,8 +105,8 @@ type Video struct {
 	Title       string           `json:"title"`
 	PublishedAt time.Time        `json:"publishedAt"`
 	Views       int              `json:"views"`
-	Tag         []string         `json:"tags,omitempty"`
+	Streamer    Streamer         `json:"streamer"`
-	Streamer    MinifiedStreamer `json:"streamer"`
+	Id          string           `json:"id"`
 }
 
 type Shelve struct {

extractor/twitch/PlaybackAccessToken.go

@@ -0,0 +1,75 @@
+package twitch
+
+import (
+	"fmt"
+
+	"github.com/tidwall/gjson"
+)
+
+type PlaybackAccessToken struct {
+	Signature string
+	Token     string
+}
+
+// Assumes streamer is live
+// If no vodID give ""
+func getPlaybackAccessToken(streamerName string, vodID string) (PlaybackAccessToken, error) {
+	var isVod bool
+	var isLive bool
+
+	if vodID != "" {
+		isVod = true
+		isLive = false
+	} else {
+		isVod = false
+		isLive = true
+	}
+
+	payload1 := []TwitchPayload{
+		{
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712",
+				},
+			},
+			"operationName": "PlaybackAccessToken",
+			"variables": map[string]interface{}{
+				"isLive":     isLive,
+				"login":      streamerName,
+				"isVod":      isVod,
+				"vodID":      vodID,
+				"playerType": "site",
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload1)
+	if err != nil {
+		return PlaybackAccessToken{}, err
+	}
+
+	fmt.Println(string(body))
+
+	var token string
+	var signature string
+	if isLive {
+		token = gjson.Get(string(body), "0.data.streamPlaybackAccessToken.value").String()
+		signature = gjson.Get(string(body), "0.data.streamPlaybackAccessToken.signature").String()
+		if err != nil {
+			return PlaybackAccessToken{}, err
+		}
+	} else {
+		token = gjson.Get(string(body), "0.data.videoPlaybackAccessToken.value").String()
+		signature = gjson.Get(string(body), "0.data.videoPlaybackAccessToken.signature").String()
+		if err != nil {
+			return PlaybackAccessToken{}, err
+		}
+	}
+
+	return PlaybackAccessToken{
+		Signature: signature,
+		Token:     token,
+	}, nil
+
+}

extractor/twitch/Stream.go

@@ -1,52 +1,26 @@
 package twitch
 
 import (
+	"fmt"
 	"io"
 	"net/http"
 	"strings"
-
-	"github.com/tidwall/gjson"
 )
 
 func GetStream(streamerName string) (string, error) {
-	// STAGE 1
-	// Get playback token from twitch
-	// Same request browser makes
-	payload1 := []TwitchPayload{
-		{
-			"extensions": map[string]interface{}{
-				"persistedQuery": map[string]interface{}{
-					"version":    1,
-					"sha256Hash": "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712",
-				},
-			},
-			"operationName": "PlaybackAccessToken",
-			"variables": map[string]interface{}{
-				"isLive":     true,
-				"login":      streamerName,
-				"isVod":      false,
-				"vodID":      "",
-				"playerType": "site",
-			},
-		},
-	}
 
-	_, body, err := parseResponse(payload1)
+	tokenwsig, err := getPlaybackAccessToken(streamerName, "")
-	if err != nil {
+	token := tokenwsig.Token
-		return "", err
+	signature := tokenwsig.Signature
-	}
-	// These will be needed to get playlist file from twitch
-	token := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.value").String()
-	signature := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.signature").String()
-	if err != nil {
-		return "", err
-	}
 
 	playlistUrl := "https://usher.ttvnw.net/api/channel/hls/" + strings.ToLower(streamerName) + ".m3u8"
 	params := "?sig=" + signature + "&token=" + token
 
 	req, err := http.NewRequest("GET", playlistUrl+params, nil)
 	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
+
+	fmt.Println(req.URL.String())
+
 	if err != nil {
 		return "", err
 	}
@@ -63,12 +37,12 @@ func GetStream(streamerName string) (string, error) {
 
 	// holy zooks, what the scallop??? we got the playlist, houston!!!
 	// time to proxy all the urls!!!
-	proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), false)
+	proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), false, false)
 	return proxiedPlaylist, nil
 
 }
 
-func GetSubPlaylist(rawurl string) (string, error) {
+func GetSubPlaylist(rawurl string, isVOD bool) (string, error) {
 	req, err := http.NewRequest("GET", rawurl, nil)
 
 	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
@@ -86,6 +60,6 @@ func GetSubPlaylist(rawurl string) (string, error) {
 		return "", err
 	}
 
-	proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), true)
+	proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), true, isVOD)
 	return proxiedPlaylist, nil
 }

extractor/twitch/VOD.go

@@ -0,0 +1,145 @@
+package twitch
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"safetwitch-backend/extractor"
+	"safetwitch-backend/extractor/structs"
+
+	"github.com/tidwall/gjson"
+)
+
+func GetStreamerFromVOD(vodID string) (string, error) {
+	payload := []TwitchPayload{
+		{
+			"operationName": "VodChannelLoginQuery",
+			"variables": map[string]interface{}{
+				"videoID": vodID,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "0c5feea4dad2565508828f16e53fe62614edf015159df4b3bca33423496ce78e",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return "", err
+	}
+
+	streamerName := gjson.Get(string(body), "0.data.video.owner.login").String()
+	if streamerName == "" {
+		return "", errors.New("streamernotfound")
+	}
+
+	return streamerName, nil
+}
+
+func GetVODPlaylist(vodID string) ([]byte, error) {
+	// get streamer
+	streamerName, err := GetStreamerFromVOD(vodID)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	// get playback token
+	tokens, err := getPlaybackAccessToken(streamerName, vodID)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	params := "?token=" + tokens.Token + "&sig=" + tokens.Signature
+	req, err := http.NewRequest("GET", "https://usher.ttvnw.net/vod/"+fmt.Sprint(vodID)+".m3u8"+params, nil)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	// headers
+	req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	proxiedPlaylist := ProxyPlaylistFile(string(body), false, true)
+
+	return []byte(proxiedPlaylist), nil
+}
+
+func GetVodMetadata(vodID string) (structs.Video, error) {
+	// get streamer
+	streamerName, err := GetStreamerFromVOD(vodID)
+	if err != nil {
+		return structs.Video{}, err
+	}
+
+	payload := []TwitchPayload{
+		{
+			"operationName": "VideoMetadata",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+				"videoID":      vodID,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "49b5b8f268cdeb259d75b58dcb0c1a748e3b575003448a2333dc5cdafd49adad",
+				},
+			},
+		},
+		{
+			"operationName": "ChannelRoot_AboutPanel",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+				"skipSchedule": false,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
+				},
+			},
+		},
+	}
+
+	_, body, err := parseResponse(payload)
+	if err != nil {
+		return structs.Video{}, err
+	}
+
+	streamerData := gjson.Get(string(body), "1.data")
+	parsedSocials, err := ParseSocials(streamerData.String())
+	if err != nil {
+		return structs.Video{}, err
+	}
+	parsedStreamer, err := ParseStreamer(streamerData, false, parsedSocials, nil, streamerName, "")
+	if err != nil {
+		return structs.Video{}, err
+	}
+
+	videoData := gjson.Get(string(body), "0.data.video")
+
+	return structs.Video{
+		Title:       videoData.Get("title").String(),
+		Preview:     extractor.ProxyUrl(videoData.Get("previewThumbnailURL").String()),
+		Duration:    int(videoData.Get("lengthSeconds").Int()),
+		Views:       int(videoData.Get("viewCount").Int()),
+		PublishedAt: videoData.Get("publishedAt").Time(),
+		Game:        ParseMinifiedCategory(videoData.Get("game")),
+		Streamer:    parsedStreamer,
+		Id:          vodID,
+	}, nil
+
+}

extractor/twitch/VODPreview.go

@@ -13,12 +13,12 @@ and look under the videos tag. Each row is considered
 a shelve, and each can be expanded. This is to be used
 on a streamers profile page
 */
-func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) {
+func GetStreamerVideoShelves(streamerName string) ([]structs.Shelve, error) {
 	payload := []TwitchPayload{
 		{
 			"operationName": "ChannelVideoShelvesQuery",
 			"variables": map[string]interface{}{
-				"channelLogin": channelName,
+				"channelLogin": streamerName,
 				"first":        5,
 			},
 			"extensions": map[string]interface{}{
@@ -28,6 +28,19 @@ func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) {
 				},
 			},
 		},
+		{
+			"operationName": "ChannelRoot_AboutPanel",
+			"variables": map[string]interface{}{
+				"channelLogin": streamerName,
+				"skipSchedule": false,
+			},
+			"extensions": map[string]interface{}{
+				"persistedQuery": map[string]interface{}{
+					"version":    1,
+					"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
+				},
+			},
+		},
 	}
 
 	_, body, err := parseResponse(payload)
@@ -35,6 +48,15 @@ func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) {
 		return []structs.Shelve{}, err
 	}
 
+	streamerData := gjson.Get(string(body), "1.data")
+	parsedSocials, err := ParseSocials(streamerData.String())
+	if err != nil {
+		return []structs.Shelve{}, err
+	}
+	parsedStreamer, err := ParseStreamer(streamerData, false, parsedSocials, nil, streamerName, "")
+	if err != nil {
+		return []structs.Shelve{}, err
+	}
 	/*
 		Twitch seperates videos by shelves.
 		There are 4 shells:
@@ -49,7 +71,7 @@ func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) {
 	shelves := gjson.Get(string(body), "0.data.user.videoShelves.edges")
 	parsedShelves := []structs.Shelve{}
 	for _, shelve := range shelves.Array() {
-		parsedShelves = append(parsedShelves, ParseShelve(shelve.Get("node")))
+		parsedShelves = append(parsedShelves, ParseShelve(shelve.Get("node"), parsedStreamer))
 	}
 
 	return parsedShelves, nil

extractor/twitch/parser.go

@@ -174,11 +174,15 @@ func ParseBadges(data gjson.Result) ([]structs.Badge, error) {
 	return formattedBadges, nil
 }
 
-func StreamSubProxyUrl(url string) string {
+func StreamSubProxyUrl(url string, isVOD bool) string {
 	encodedUrl := base64.StdEncoding.EncodeToString([]byte(url))
 	backendUrl := os.Getenv("URL")
 
-	return backendUrl + "/proxy/stream/sub/" + encodedUrl
+	if isVOD {
+		return backendUrl + "/proxy/vod/sub/" + encodedUrl + "/video.m3u8"
+	} else {
+		return backendUrl + "/proxy/stream/sub/" + encodedUrl
+	}
 }
 
 func StreamSegmentProxyUrl(url string) string {
@@ -188,7 +192,7 @@ func StreamSegmentProxyUrl(url string) string {
 	return backendUrl + "/proxy/stream/segment/" + encodedUrl
 }
 
-func ProxyPlaylistFile(playlist string, isSubPlaylist bool) string {
+func ProxyPlaylistFile(playlist string, isSubPlaylist bool, isVod bool) string {
 	// Split the playlist into individual entries
 	entries := strings.Split(playlist, "\n")[1:] // Ignore the first line which contains the M3U header
 
@@ -199,17 +203,25 @@ func ProxyPlaylistFile(playlist string, isSubPlaylist bool) string {
 			if isSubPlaylist {
 				newURL = StreamSegmentProxyUrl(entry)
 			} else {
-				newURL = StreamSubProxyUrl(entry)
+				newURL = StreamSubProxyUrl(entry, isVod)
 			}
 			entries[i] = newURL
 		}
 	}
 
+	if isVod {
+		// Loop through each entry and replace the URL
+		for i, entry := range entries {
+			if strings.HasSuffix(entry, ".ts") {
+				entries[i] = entry
+			}
+		}
+	}
+
 	// Join the modified entries back into a single string separated by a newline
 	modifiedPlaylist := "#EXTM3U\n" + strings.Join(entries, "\n")
 
 	return modifiedPlaylist
-
 }
 
 func ParseMinifiedCategory(data gjson.Result) structs.MinifiedCategory {
@@ -230,13 +242,7 @@ func ParseMinifiedStreamer(data gjson.Result) structs.MinifiedStreamer {
 	}
 }
 
-func ParseVideo(data gjson.Result) structs.Video {
+func ParseVideo(data gjson.Result, streamer structs.Streamer) structs.Video {
-	tags := []string{}
-
-	for _, tag := range data.Get("contentTags").Array() {
-		tags = append(tags, tag.Get("localizedName").String())
-	}
-
 	return structs.Video{
 		Preview:     extractor.ProxyUrl(data.Get("previewThumbnailURL").String()),
 		Game:        ParseMinifiedCategory(data.Get("game")),
@@ -244,18 +250,12 @@ func ParseVideo(data gjson.Result) structs.Video {
 		Title:       data.Get("title").String(),
 		PublishedAt: data.Get("createdAt").Time(),
 		Views:       int(data.Get("viewCount").Int()),
-		Tag:         tags,
+		Streamer:    streamer,
-		Streamer:    ParseMinifiedStreamer(data.Get("owner")),
+		Id:          data.Get("id").String(),
 	}
 }
 
-func ParseClip(data gjson.Result) structs.Video {
+func ParseClip(data gjson.Result, streamer structs.Streamer) structs.Video {
-	tags := []string{}
-
-	for _, tag := range data.Get("contentTags").Array() {
-		tags = append(tags, tag.Get("localizedName").String())
-	}
-
 	return structs.Video{
 		Preview:     extractor.ProxyUrl(data.Get("thumbnailURL").String()),
 		Game:        ParseMinifiedCategory(data.Get("clipGame")),
@@ -263,20 +263,20 @@ func ParseClip(data gjson.Result) structs.Video {
 		Title:       data.Get("clipTitle").String(),
 		PublishedAt: data.Get("publishedAt").Time(),
 		Views:       int(data.Get("clipViewCount").Int()),
-		Tag:         tags,
+		Streamer:    streamer,
-		Streamer:    ParseMinifiedStreamer(data.Get("broadcaster")),
+		Id:          data.Get("id").String(),
 	}
 }
-func ParseShelve(data gjson.Result) structs.Shelve {
+func ParseShelve(data gjson.Result, streamer structs.Streamer) structs.Shelve {
 	rawVideos := data.Get("items").Array()
 	parsedVideos := []structs.Video{}
 
 	isClip := data.Get("type").String() == "TOP_CLIPS"
 	for _, video := range rawVideos {
 		if isClip {
-			parsedVideos = append(parsedVideos, ParseClip(video))
+			parsedVideos = append(parsedVideos, ParseClip(video, streamer))
 		} else {
-			parsedVideos = append(parsedVideos, ParseVideo(video))
+			parsedVideos = append(parsedVideos, ParseVideo(video, streamer))
 		}
 	}
 

routes/api/vods/homeShelves.go

@@ -1,17 +0,0 @@
-package vods
-
-import (
-	"safetwitch-backend/extractor"
-	"safetwitch-backend/extractor/twitch"
-
-	"github.com/gin-gonic/gin"
-)
-
-func Routes(route *gin.Engine) {
-	auth := route.Group("/api/vods")
-
-	auth.GET("/shelve/:streamerName", func(context *gin.Context) {
-		data, _ := twitch.GetStreamerVideoShelves(context.Param("streamerName"))
-		context.JSON(200, extractor.FormatMessage(data, true))
-	})
-}

routes/api/vods/vods.go

@@ -0,0 +1,30 @@
+package vods
+
+import (
+	"safetwitch-backend/extractor"
+	"safetwitch-backend/extractor/twitch"
+
+	"github.com/gin-gonic/gin"
+)
+
+func Routes(route *gin.Engine) {
+	auth := route.Group("/api/vods")
+
+	auth.GET("/shelve/:streamerName", func(context *gin.Context) {
+		data, err := twitch.GetStreamerVideoShelves(context.Param("streamerName"))
+		if err != nil {
+			context.Error(err)
+			return
+		}
+		context.JSON(200, extractor.FormatMessage(data, true))
+	})
+
+	auth.GET("/:streamerName", func(context *gin.Context) {
+		data, err := twitch.GetVodMetadata(context.Param("streamerName"))
+		if err != nil {
+			context.Error(err)
+			return
+		}
+		context.JSON(200, extractor.FormatMessage(data, true))
+	})
+}

routes/proxy/proxy.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"net/http"
 	"safetwitch-backend/extractor/twitch"
+	"strings"
 
 	"github.com/gin-gonic/gin"
 )
@@ -40,7 +41,7 @@ func Routes(route *gin.Engine) {
 			context.Error(err)
 		}
 
-		context.Data(200, "application/text", []byte(playlistFile))
+		context.Data(200, "application/vnd.apple.mpegurl", []byte(playlistFile))
 	})
 
 	auth.GET("/stream/sub/:encodedUrl", func(context *gin.Context) {
@@ -49,12 +50,12 @@ func Routes(route *gin.Engine) {
 			context.Error(err)
 		}
 
-		playlistFile, err := twitch.GetSubPlaylist(string(decodedUrl))
+		playlistFile, err := twitch.GetSubPlaylist(string(decodedUrl), false)
 		if err != nil {
 			context.Error(err)
 		}
 
-		context.Data(200, "application/text", []byte(playlistFile))
+		context.Data(200, "application/vnd.apple.mpegurl", []byte(playlistFile))
 	})
 
 	auth.GET("/stream/segment/:encodedUrl", func(context *gin.Context) {
@@ -74,6 +75,55 @@ func Routes(route *gin.Engine) {
 			context.Error(err)
 		}
 
-		context.Data(200, "fuck", segment)
+		context.Data(200, "application/text", segment)
+	})
+
+	// vod
+	auth.GET("/vod/:vodID/video.m3u8", func(context *gin.Context) {
+		vodID := context.Param("vodID")
+		data, err := twitch.GetVODPlaylist(vodID)
+		if err != nil {
+			context.Error(err)
+			return
+		}
+		context.Data(200, "application/vnd.apple.mpegurl", data)
+	})
+
+	auth.GET("/vod/sub/:encodedUrl/video.m3u8", func(context *gin.Context) {
+		decodedUrl, err := b64.StdEncoding.DecodeString(context.Param("encodedUrl"))
+		if err != nil {
+			context.Error(err)
+		}
+
+		playlistFile, err := twitch.GetSubPlaylist(string(decodedUrl), true)
+		if err != nil {
+			context.Error(err)
+		}
+
+		context.Data(200, "application/vnd.apple.mpegurl", []byte(playlistFile))
+	})
+
+	auth.GET("/vod/sub/:encodedUrl/:segment", func(context *gin.Context) {
+		decodedUrl, err := b64.StdEncoding.DecodeString(context.Param("encodedUrl"))
+		if err != nil {
+			context.Error(err)
+		}
+
+		// remove the last path of url and replace with segment
+		tempurl := strings.Split(string(decodedUrl), "/")
+		newurl := strings.Join(tempurl[:len(tempurl)-1], "/") + "/" + context.Param("segment")
+
+		segmentData, err := http.Get(string(newurl))
+		if err != nil {
+			context.Error(err)
+			return
+		}
+
+		segment, err := io.ReadAll(segmentData.Body)
+		if err != nil {
+			context.Error(err)
+		}
+
+		context.Data(200, "application/text", segment)
 	})
 }