Infrastructure/safetwitch-backend
0
Fork 0
mirror of https://codeberg.org/SafeTwitch/safetwitch-backend.git synced 2024-12-22 05:02:58 -05:00
Code Issues Activity

VOD support

Browse source
This commit is contained in:
dragongoose 2023-07-17 13:16:33 -04:00
parent 87dc0b8d2c
commit 16c8402936
No known key found for this signature in database
GPG key ID: 01397EEC371CDAA5
11 changed files with 392 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
README.md
View file

@ -327,3 +327,28 @@ Gets a segment from one of the quality's manifest file. This is the actual video
### 200
Returns the stream segment, HLS streaming.
### /proxy/vod/:vodID/video.m3u8
**GET**
Gets the master manifest for a VOD
### 200
Returns the manifest file
### /proxy/vod/sub/:encodedUrl/video.m3u8
**GET**
Gets the sub manifest for a VOD
encodedUrl is the url to the sub manifiest from twitch encoded in base64
### 200
Returns the manifest file
### /proxy/vod/sub/:encodedUrl/:segment
**GET**
Gets the sub manifest for a VOD
encodedUrl is the url to the sub manifiest from twitch encoded in base64
segment is the segment from the playlist file, etc 0.ts, 1.ts
### 200
Returns the manifest file

4
extractor/structs/parsed.go
View file

@ -105,8 +105,8 @@ type Video struct {
Title string `json:"title"`
PublishedAt time.Time `json:"publishedAt"`
Views int `json:"views"`
Tag []string `json:"tags,omitempty"`
Streamer MinifiedStreamer `json:"streamer"`
Streamer Streamer `json:"streamer"`
Id string `json:"id"`
}
type Shelve struct {

75
extractor/twitch/PlaybackAccessToken.go Normal file
View file

@ -0,0 +1,75 @@
package twitch
import (
"fmt"
"github.com/tidwall/gjson"
)
type PlaybackAccessToken struct {
Signature string
Token string
}
// Assumes streamer is live
// If no vodID give ""
func getPlaybackAccessToken(streamerName string, vodID string) (PlaybackAccessToken, error) {
var isVod bool
var isLive bool
if vodID != "" {
isVod = true
isLive = false
} else {
isVod = false
isLive = true
}
payload1 := []TwitchPayload{
{
"extensions": map[string]interface{}{
"persistedQuery": map[string]interface{}{
"version": 1,
"sha256Hash": "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712",
},
},
"operationName": "PlaybackAccessToken",
"variables": map[string]interface{}{
"isLive": isLive,
"login": streamerName,
"isVod": isVod,
"vodID": vodID,
"playerType": "site",
},
},
}
_, body, err := parseResponse(payload1)
if err != nil {
return PlaybackAccessToken{}, err
}
fmt.Println(string(body))
var token string
var signature string
if isLive {
token = gjson.Get(string(body), "0.data.streamPlaybackAccessToken.value").String()
signature = gjson.Get(string(body), "0.data.streamPlaybackAccessToken.signature").String()
if err != nil {
return PlaybackAccessToken{}, err
}
} else {
token = gjson.Get(string(body), "0.data.videoPlaybackAccessToken.value").String()
signature = gjson.Get(string(body), "0.data.videoPlaybackAccessToken.signature").String()
if err != nil {
return PlaybackAccessToken{}, err
}
}
return PlaybackAccessToken{
Signature: signature,
Token: token,
}, nil
}

46
extractor/twitch/Stream.go
View file

@ -1,52 +1,26 @@
package twitch
import (
"fmt"
"io"
"net/http"
"strings"
"github.com/tidwall/gjson"
)
func GetStream(streamerName string) (string, error) {
// STAGE 1
// Get playback token from twitch
// Same request browser makes
payload1 := []TwitchPayload{
{
"extensions": map[string]interface{}{
"persistedQuery": map[string]interface{}{
"version": 1,
"sha256Hash": "0828119ded1c13477966434e15800ff57ddacf13ba1911c129dc2200705b0712",
},
},
"operationName": "PlaybackAccessToken",
"variables": map[string]interface{}{
"isLive": true,
"login": streamerName,
"isVod": false,
"vodID": "",
"playerType": "site",
},
},
}
_, body, err := parseResponse(payload1)
if err != nil {
return "", err
}
// These will be needed to get playlist file from twitch
token := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.value").String()
signature := gjson.Get(string(body), "0.data.streamPlaybackAccessToken.signature").String()
if err != nil {
return "", err
}
tokenwsig, err := getPlaybackAccessToken(streamerName, "")
token := tokenwsig.Token
signature := tokenwsig.Signature
playlistUrl := "https://usher.ttvnw.net/api/channel/hls/" + strings.ToLower(streamerName) + ".m3u8"
params := "?sig=" + signature + "&token=" + token
req, err := http.NewRequest("GET", playlistUrl+params, nil)
req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
fmt.Println(req.URL.String())
if err != nil {
return "", err
}
@ -63,12 +37,12 @@ func GetStream(streamerName string) (string, error) {
// holy zooks, what the scallop??? we got the playlist, houston!!!
// time to proxy all the urls!!!
proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), false)
proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), false, false)
return proxiedPlaylist, nil
}
func GetSubPlaylist(rawurl string) (string, error) {
func GetSubPlaylist(rawurl string, isVOD bool) (string, error) {
req, err := http.NewRequest("GET", rawurl, nil)
req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
@ -86,6 +60,6 @@ func GetSubPlaylist(rawurl string) (string, error) {
return "", err
}
proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), true)
proxiedPlaylist := ProxyPlaylistFile(string(playlistFile), true, isVOD)
return proxiedPlaylist, nil
}

145
extractor/twitch/VOD.go Normal file
View file

@ -0,0 +1,145 @@
package twitch
import (
"errors"
"fmt"
"io"
"net/http"
"safetwitch-backend/extractor"
"safetwitch-backend/extractor/structs"
"github.com/tidwall/gjson"
)
func GetStreamerFromVOD(vodID string) (string, error) {
payload := []TwitchPayload{
{
"operationName": "VodChannelLoginQuery",
"variables": map[string]interface{}{
"videoID": vodID,
},
"extensions": map[string]interface{}{
"persistedQuery": map[string]interface{}{
"version": 1,
"sha256Hash": "0c5feea4dad2565508828f16e53fe62614edf015159df4b3bca33423496ce78e",
},
},
},
}
_, body, err := parseResponse(payload)
if err != nil {
return "", err
}
streamerName := gjson.Get(string(body), "0.data.video.owner.login").String()
if streamerName == "" {
return "", errors.New("streamernotfound")
}
return streamerName, nil
}
func GetVODPlaylist(vodID string) ([]byte, error) {
// get streamer
streamerName, err := GetStreamerFromVOD(vodID)
if err != nil {
return []byte{}, err
}
// get playback token
tokens, err := getPlaybackAccessToken(streamerName, vodID)
if err != nil {
return []byte{}, err
}
params := "?token=" + tokens.Token + "&sig=" + tokens.Signature
req, err := http.NewRequest("GET", "https://usher.ttvnw.net/vod/"+fmt.Sprint(vodID)+".m3u8"+params, nil)
if err != nil {
return []byte{}, err
}
// headers
req.Header.Add("Client-Id", "ue6666qo983tsx6so1t0vnawi233wa")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return []byte{}, err
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return []byte{}, err
}
proxiedPlaylist := ProxyPlaylistFile(string(body), false, true)
return []byte(proxiedPlaylist), nil
}
func GetVodMetadata(vodID string) (structs.Video, error) {
// get streamer
streamerName, err := GetStreamerFromVOD(vodID)
if err != nil {
return structs.Video{}, err
}
payload := []TwitchPayload{
{
"operationName": "VideoMetadata",
"variables": map[string]interface{}{
"channelLogin": streamerName,
"videoID": vodID,
},
"extensions": map[string]interface{}{
"persistedQuery": map[string]interface{}{
"version": 1,
"sha256Hash": "49b5b8f268cdeb259d75b58dcb0c1a748e3b575003448a2333dc5cdafd49adad",
},
},
},
{
"operationName": "ChannelRoot_AboutPanel",
"variables": map[string]interface{}{
"channelLogin": streamerName,
"skipSchedule": false,
},
"extensions": map[string]interface{}{
"persistedQuery": map[string]interface{}{
"version": 1,
"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
},
},
},
}
_, body, err := parseResponse(payload)
if err != nil {
return structs.Video{}, err
}
streamerData := gjson.Get(string(body), "1.data")
parsedSocials, err := ParseSocials(streamerData.String())
if err != nil {
return structs.Video{}, err
}
parsedStreamer, err := ParseStreamer(streamerData, false, parsedSocials, nil, streamerName, "")
if err != nil {
return structs.Video{}, err
}
videoData := gjson.Get(string(body), "0.data.video")
return structs.Video{
Title: videoData.Get("title").String(),
Preview: extractor.ProxyUrl(videoData.Get("previewThumbnailURL").String()),
Duration: int(videoData.Get("lengthSeconds").Int()),
Views: int(videoData.Get("viewCount").Int()),
PublishedAt: videoData.Get("publishedAt").Time(),
Game: ParseMinifiedCategory(videoData.Get("game")),
Streamer: parsedStreamer,
Id: vodID,
}, nil
}

28
extractor/twitch/VODPreview.go
View file

@ -13,12 +13,12 @@ and look under the videos tag. Each row is considered
a shelve, and each can be expanded. This is to be used
on a streamers profile page
*/
func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) {
func GetStreamerVideoShelves(streamerName string) ([]structs.Shelve, error) {
payload := []TwitchPayload{
{
"operationName": "ChannelVideoShelvesQuery",
"variables": map[string]interface{}{
"channelLogin": channelName,
"channelLogin": streamerName,
"first": 5,
},
"extensions": map[string]interface{}{
@ -28,6 +28,19 @@ func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) {
},
},
},
{
"operationName": "ChannelRoot_AboutPanel",
"variables": map[string]interface{}{
"channelLogin": streamerName,
"skipSchedule": false,
},
"extensions": map[string]interface{}{
"persistedQuery": map[string]interface{}{
"version": 1,
"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6",
},
},
},
}
_, body, err := parseResponse(payload)
@ -35,6 +48,15 @@ func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) {
return []structs.Shelve{}, err
}
streamerData := gjson.Get(string(body), "1.data")
parsedSocials, err := ParseSocials(streamerData.String())
if err != nil {
return []structs.Shelve{}, err
}
parsedStreamer, err := ParseStreamer(streamerData, false, parsedSocials, nil, streamerName, "")
if err != nil {
return []structs.Shelve{}, err
}
/*
Twitch seperates videos by shelves.
There are 4 shells:
@ -49,7 +71,7 @@ func GetStreamerVideoShelves(channelName string) ([]structs.Shelve, error) {
shelves := gjson.Get(string(body), "0.data.user.videoShelves.edges")
parsedShelves := []structs.Shelve{}
for _, shelve := range shelves.Array() {
parsedShelves = append(parsedShelves, ParseShelve(shelve.Get("node")))
parsedShelves = append(parsedShelves, ParseShelve(shelve.Get("node"), parsedStreamer))
}
return parsedShelves, nil

52
extractor/twitch/parser.go
View file

@ -174,11 +174,15 @@ func ParseBadges(data gjson.Result) ([]structs.Badge, error) {
return formattedBadges, nil
}
func StreamSubProxyUrl(url string) string {
func StreamSubProxyUrl(url string, isVOD bool) string {
encodedUrl := base64.StdEncoding.EncodeToString([]byte(url))
backendUrl := os.Getenv("URL")
return backendUrl + "/proxy/stream/sub/" + encodedUrl
if isVOD {
return backendUrl + "/proxy/vod/sub/" + encodedUrl + "/video.m3u8"
} else {
return backendUrl + "/proxy/stream/sub/" + encodedUrl
}
}
func StreamSegmentProxyUrl(url string) string {
@ -188,7 +192,7 @@ func StreamSegmentProxyUrl(url string) string {
return backendUrl + "/proxy/stream/segment/" + encodedUrl
}
func ProxyPlaylistFile(playlist string, isSubPlaylist bool) string {
func ProxyPlaylistFile(playlist string, isSubPlaylist bool, isVod bool) string {
// Split the playlist into individual entries
entries := strings.Split(playlist, "\n")[1:] // Ignore the first line which contains the M3U header
@ -199,17 +203,25 @@ func ProxyPlaylistFile(playlist string, isSubPlaylist bool) string {
if isSubPlaylist {
newURL = StreamSegmentProxyUrl(entry)
} else {
newURL = StreamSubProxyUrl(entry)
newURL = StreamSubProxyUrl(entry, isVod)
}
entries[i] = newURL
}
}
if isVod {
// Loop through each entry and replace the URL
for i, entry := range entries {
if strings.HasSuffix(entry, ".ts") {
entries[i] = entry
}
}
}
// Join the modified entries back into a single string separated by a newline
modifiedPlaylist := "#EXTM3U\n" + strings.Join(entries, "\n")
return modifiedPlaylist
}
func ParseMinifiedCategory(data gjson.Result) structs.MinifiedCategory {
@ -230,13 +242,7 @@ func ParseMinifiedStreamer(data gjson.Result) structs.MinifiedStreamer {
}
}
func ParseVideo(data gjson.Result) structs.Video {
tags := []string{}
for _, tag := range data.Get("contentTags").Array() {
tags = append(tags, tag.Get("localizedName").String())
}
func ParseVideo(data gjson.Result, streamer structs.Streamer) structs.Video {
return structs.Video{
Preview: extractor.ProxyUrl(data.Get("previewThumbnailURL").String()),
Game: ParseMinifiedCategory(data.Get("game")),
@ -244,18 +250,12 @@ func ParseVideo(data gjson.Result) structs.Video {
Title: data.Get("title").String(),
PublishedAt: data.Get("createdAt").Time(),
Views: int(data.Get("viewCount").Int()),
Tag: tags,
Streamer: ParseMinifiedStreamer(data.Get("owner")),
Streamer: streamer,
Id: data.Get("id").String(),
}
}
func ParseClip(data gjson.Result) structs.Video {
tags := []string{}
for _, tag := range data.Get("contentTags").Array() {
tags = append(tags, tag.Get("localizedName").String())
}
func ParseClip(data gjson.Result, streamer structs.Streamer) structs.Video {
return structs.Video{
Preview: extractor.ProxyUrl(data.Get("thumbnailURL").String()),
Game: ParseMinifiedCategory(data.Get("clipGame")),
@ -263,20 +263,20 @@ func ParseClip(data gjson.Result) structs.Video {
Title: data.Get("clipTitle").String(),
PublishedAt: data.Get("publishedAt").Time(),
Views: int(data.Get("clipViewCount").Int()),
Tag: tags,
Streamer: ParseMinifiedStreamer(data.Get("broadcaster")),
Streamer: streamer,
Id: data.Get("id").String(),
}
}
func ParseShelve(data gjson.Result) structs.Shelve {
func ParseShelve(data gjson.Result, streamer structs.Streamer) structs.Shelve {
rawVideos := data.Get("items").Array()
parsedVideos := []structs.Video{}
isClip := data.Get("type").String() == "TOP_CLIPS"
for _, video := range rawVideos {
if isClip {
parsedVideos = append(parsedVideos, ParseClip(video))
parsedVideos = append(parsedVideos, ParseClip(video, streamer))
} else {
parsedVideos = append(parsedVideos, ParseVideo(video))
parsedVideos = append(parsedVideos, ParseVideo(video, streamer))
}
}

0
extractor/twitch/utils.go → extractor/twitch/twitchExtractor.go
View file

17
routes/api/vods/homeShelves.go
View file

@ -1,17 +0,0 @@
package vods
import (
"safetwitch-backend/extractor"
"safetwitch-backend/extractor/twitch"
"github.com/gin-gonic/gin"
)
func Routes(route *gin.Engine) {
auth := route.Group("/api/vods")
auth.GET("/shelve/:streamerName", func(context *gin.Context) {
data, _ := twitch.GetStreamerVideoShelves(context.Param("streamerName"))
context.JSON(200, extractor.FormatMessage(data, true))
})
}

30
routes/api/vods/vods.go Normal file
View file

@ -0,0 +1,30 @@
package vods
import (
"safetwitch-backend/extractor"
"safetwitch-backend/extractor/twitch"
"github.com/gin-gonic/gin"
)
func Routes(route *gin.Engine) {
auth := route.Group("/api/vods")
auth.GET("/shelve/:streamerName", func(context *gin.Context) {
data, err := twitch.GetStreamerVideoShelves(context.Param("streamerName"))
if err != nil {
context.Error(err)
return
}
context.JSON(200, extractor.FormatMessage(data, true))
})
auth.GET("/:streamerName", func(context *gin.Context) {
data, err := twitch.GetVodMetadata(context.Param("streamerName"))
if err != nil {
context.Error(err)
return
}
context.JSON(200, extractor.FormatMessage(data, true))
})
}

58
routes/proxy/proxy.go
View file

@ -5,6 +5,7 @@ import (
"io"
"net/http"
"safetwitch-backend/extractor/twitch"
"strings"
"github.com/gin-gonic/gin"
)
@ -40,7 +41,7 @@ func Routes(route *gin.Engine) {
context.Error(err)
}
context.Data(200, "application/text", []byte(playlistFile))
context.Data(200, "application/vnd.apple.mpegurl", []byte(playlistFile))
})
auth.GET("/stream/sub/:encodedUrl", func(context *gin.Context) {
@ -49,12 +50,12 @@ func Routes(route *gin.Engine) {
context.Error(err)
}
playlistFile, err := twitch.GetSubPlaylist(string(decodedUrl))
playlistFile, err := twitch.GetSubPlaylist(string(decodedUrl), false)
if err != nil {
context.Error(err)
}
context.Data(200, "application/text", []byte(playlistFile))
context.Data(200, "application/vnd.apple.mpegurl", []byte(playlistFile))
})
auth.GET("/stream/segment/:encodedUrl", func(context *gin.Context) {
@ -74,6 +75,55 @@ func Routes(route *gin.Engine) {
context.Error(err)
}
context.Data(200, "fuck", segment)
context.Data(200, "application/text", segment)
})
// vod
auth.GET("/vod/:vodID/video.m3u8", func(context *gin.Context) {
vodID := context.Param("vodID")
data, err := twitch.GetVODPlaylist(vodID)
if err != nil {
context.Error(err)
return
}
context.Data(200, "application/vnd.apple.mpegurl", data)
})
auth.GET("/vod/sub/:encodedUrl/video.m3u8", func(context *gin.Context) {
decodedUrl, err := b64.StdEncoding.DecodeString(context.Param("encodedUrl"))
if err != nil {
context.Error(err)
}
playlistFile, err := twitch.GetSubPlaylist(string(decodedUrl), true)
if err != nil {
context.Error(err)
}
context.Data(200, "application/vnd.apple.mpegurl", []byte(playlistFile))
})
auth.GET("/vod/sub/:encodedUrl/:segment", func(context *gin.Context) {
decodedUrl, err := b64.StdEncoding.DecodeString(context.Param("encodedUrl"))
if err != nil {
context.Error(err)
}
// remove the last path of url and replace with segment
tempurl := strings.Split(string(decodedUrl), "/")
newurl := strings.Join(tempurl[:len(tempurl)-1], "/") + "/" + context.Param("segment")
segmentData, err := http.Get(string(newurl))
if err != nil {
context.Error(err)
return
}
segment, err := io.ReadAll(segmentData.Body)
if err != nil {
context.Error(err)
}
context.Data(200, "application/text", segment)
})
}