mirror of
https://codeberg.org/SafeTwitch/safetwitch-backend.git
synced 2024-12-22 13:13:00 -05:00
Initial Commit
This commit is contained in:
commit
b42f7b189f
17 changed files with 5573 additions and 0 deletions
2
.env
Normal file
2
.env
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
PORT=7000
|
||||||
|
URL=http://localhost:7000
|
3
.gitignore
vendored
Normal file
3
.gitignore
vendored
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
node_modules
|
||||||
|
puppeteerData
|
||||||
|
serverLog.log
|
33
index.ts
Normal file
33
index.ts
Normal file
|
@ -0,0 +1,33 @@
|
||||||
|
import express, { Express } from 'express';
|
||||||
|
import dotenv from 'dotenv'
|
||||||
|
import routes from './routes'
|
||||||
|
import { errorHandler, uuid } from './util/logger'
|
||||||
|
import { wsServer } from './routes/proxyRoute';
|
||||||
|
|
||||||
|
dotenv.config()
|
||||||
|
|
||||||
|
const app: Express = express();
|
||||||
|
const port = process.env.PORT
|
||||||
|
|
||||||
|
app.use(uuid)
|
||||||
|
app.use(routes)
|
||||||
|
|
||||||
|
// 404 handler
|
||||||
|
app.use((req, res) => {
|
||||||
|
if (!res.headersSent) {
|
||||||
|
res.status(404).send('404')
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// handle errors
|
||||||
|
app.use(errorHandler)
|
||||||
|
|
||||||
|
const server = app.listen(port, () => {
|
||||||
|
console.log('Server up')
|
||||||
|
})
|
||||||
|
|
||||||
|
server.on('upgrade', (request, socket, head) => {
|
||||||
|
wsServer.handleUpgrade(request, socket, head, socket => {
|
||||||
|
wsServer.emit('connection', socket, request);
|
||||||
|
});
|
||||||
|
});
|
4629
package-lock.json
generated
Normal file
4629
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
24
package.json
Normal file
24
package.json
Normal file
|
@ -0,0 +1,24 @@
|
||||||
|
{
|
||||||
|
"dependencies": {
|
||||||
|
"connect-history-api-fallback": "^2.0.0",
|
||||||
|
"dotenv": "^16.0.3",
|
||||||
|
"express": "^4.18.2",
|
||||||
|
"videojs-contrib-quality-levels": "^3.0.0",
|
||||||
|
"videojs-hls-quality-selector": "^1.1.4",
|
||||||
|
"winston": "^3.8.2",
|
||||||
|
"ws": "^8.13.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/connect-history-api-fallback": "^1.3.5",
|
||||||
|
"@types/express": "^4.17.17",
|
||||||
|
"@types/node": "^18.14.6",
|
||||||
|
"@types/ws": "^8.5.4",
|
||||||
|
"nodemon": "^2.0.21",
|
||||||
|
"ts-node": "^10.9.1",
|
||||||
|
"typescript": "^4.9.5"
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"dev": "npx nodemon index.ts",
|
||||||
|
"prod": "npx ts-node index.ts"
|
||||||
|
}
|
||||||
|
}
|
2
readme.md
Normal file
2
readme.md
Normal file
|
@ -0,0 +1,2 @@
|
||||||
|
# twitch-backend
|
||||||
|
The backend for [twitch-frontend](https://github.com/dragongoose/twitch-frontend)
|
10
routes.ts
Normal file
10
routes.ts
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
import { Router } from 'express';
|
||||||
|
import profileRoute from './routes/profileRoute'
|
||||||
|
import proxyRoute from './routes/proxyRoute'
|
||||||
|
|
||||||
|
const routes = Router();
|
||||||
|
|
||||||
|
routes.use('/api', profileRoute)
|
||||||
|
routes.use('/proxy', proxyRoute)
|
||||||
|
|
||||||
|
export default routes
|
27
routes/profileRoute.ts
Normal file
27
routes/profileRoute.ts
Normal file
|
@ -0,0 +1,27 @@
|
||||||
|
import { Router } from 'express'
|
||||||
|
import { TwitchAPI } from '../util/scraping/extractor/index'
|
||||||
|
|
||||||
|
const profileRouter = Router()
|
||||||
|
const twitch = new TwitchAPI()
|
||||||
|
|
||||||
|
profileRouter.get('/users/:username', async (req, res, next) => {
|
||||||
|
const username = req.params.username
|
||||||
|
|
||||||
|
let streamerData = await twitch.getStreamerInfo(username)
|
||||||
|
.catch(next)
|
||||||
|
|
||||||
|
if (streamerData)
|
||||||
|
res.send(streamerData)
|
||||||
|
})
|
||||||
|
|
||||||
|
profileRouter.get('/discover', async (req, res, next) => {
|
||||||
|
let discoveryData = await twitch.getDirectory(50)
|
||||||
|
res.send(discoveryData)
|
||||||
|
})
|
||||||
|
|
||||||
|
profileRouter.get('/discover/:game', async (req, res, next) => {
|
||||||
|
let discoveryData = await twitch.getDirectoryGame(req.params.game, 50)
|
||||||
|
res.send(discoveryData)
|
||||||
|
})
|
||||||
|
|
||||||
|
export default profileRouter
|
131
routes/proxyRoute.ts
Normal file
131
routes/proxyRoute.ts
Normal file
|
@ -0,0 +1,131 @@
|
||||||
|
import { Router, Response, Request, NextFunction } from 'express'
|
||||||
|
import { TwitchAPI } from '../util/scraping/extractor';
|
||||||
|
import ws, { WebSocket } from 'ws';
|
||||||
|
import { TwitchChat } from '../util/scraping/chat/chat';
|
||||||
|
import { logger } from '../util/logger';
|
||||||
|
|
||||||
|
const proxyRouter = Router();
|
||||||
|
const twitch = new TwitchAPI()
|
||||||
|
|
||||||
|
proxyRouter.get('/img/:base64Url', async (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
const imageUrl = Buffer.from(req.params.base64Url, 'base64url').toString('utf-8')
|
||||||
|
console.log(imageUrl)
|
||||||
|
if(!imageUrl) return;
|
||||||
|
|
||||||
|
const imageRes = await fetch(imageUrl)
|
||||||
|
.catch(next)
|
||||||
|
|
||||||
|
if(!imageRes) return;
|
||||||
|
|
||||||
|
if(imageRes.status !== 200) {
|
||||||
|
res.status(imageRes.status).send()
|
||||||
|
const error = new Error('Image proxy fetch was not status 200')
|
||||||
|
logger.warn(error)
|
||||||
|
}
|
||||||
|
|
||||||
|
const arrayBuffer = await imageRes.arrayBuffer()
|
||||||
|
const buf = Buffer.from(arrayBuffer)
|
||||||
|
|
||||||
|
res.send(buf)
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
proxyRouter.get('/stream/:username/hls.m3u8', async (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
let m3u8Data = await twitch.getStream(req.params.username)
|
||||||
|
const urlRegex =/(\b(https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])/ig;
|
||||||
|
const matches = m3u8Data.match(urlRegex)
|
||||||
|
if (!matches) return next(new Error('Error proxying HLS'));
|
||||||
|
|
||||||
|
for (let url of matches) {
|
||||||
|
const base64data = Buffer.from(url).toString('base64url')
|
||||||
|
m3u8Data = m3u8Data.replace(url, `${process.env.URL}/proxy/hls/${base64data}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
res.setHeader('Content-type','application/vnd.apple.mpegurl')
|
||||||
|
res.send(m3u8Data)
|
||||||
|
})
|
||||||
|
|
||||||
|
proxyRouter.get('/hls/:encodedUrl' , async (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
const unencodedUrl = Buffer.from(req.params.encodedUrl, 'base64url').toString()
|
||||||
|
const m3u8Fetch = await fetch(unencodedUrl)
|
||||||
|
var m3u8Data = await m3u8Fetch.text()
|
||||||
|
|
||||||
|
res.send(m3u8Data)
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// IRC PROXY
|
||||||
|
interface ExtWebSocket extends WebSocket {
|
||||||
|
id: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const chat = new TwitchChat({
|
||||||
|
login: {
|
||||||
|
username: 'justinfan23423',
|
||||||
|
password: 'none'
|
||||||
|
},
|
||||||
|
channels: []
|
||||||
|
})
|
||||||
|
chat.connect()
|
||||||
|
|
||||||
|
const clients : { [k:string]: ExtWebSocket[] } = {}
|
||||||
|
import { randomUUID } from 'crypto';
|
||||||
|
|
||||||
|
const findClientsForStreamer = async (streamerName: string) => {
|
||||||
|
if(!clients[streamerName]) return Promise.reject(new Error('No clients following streamer'))
|
||||||
|
|
||||||
|
return clients[streamerName]
|
||||||
|
}
|
||||||
|
|
||||||
|
export const wsServer = new ws.Server({ noServer: true });
|
||||||
|
wsServer.on('connection', (ws: ExtWebSocket) => {
|
||||||
|
const socket = ws as ExtWebSocket
|
||||||
|
socket.on('message', (message) => {
|
||||||
|
const data = message.toString()
|
||||||
|
const splitted = data.split(' ')
|
||||||
|
|
||||||
|
if(splitted.length > 2) socket.close()
|
||||||
|
if(splitted[0] !== 'JOIN') socket.close()
|
||||||
|
|
||||||
|
const streamersToJoin = splitted[1].split(',')
|
||||||
|
if(streamersToJoin.length > 1) socket.close()
|
||||||
|
|
||||||
|
const id = randomUUID()
|
||||||
|
for (let streamer of streamersToJoin) {
|
||||||
|
chat.addStreamer(streamer)
|
||||||
|
|
||||||
|
if(clients[streamer]) {
|
||||||
|
clients[streamer].push(socket)
|
||||||
|
} else {
|
||||||
|
clients[streamer] = [socket]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
socket.id = id
|
||||||
|
socket.send('OK')
|
||||||
|
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('close', () => {
|
||||||
|
if(socket.id) {
|
||||||
|
}
|
||||||
|
})
|
||||||
|
});
|
||||||
|
|
||||||
|
chat.on('PRIVMSG', async (username, type, channel, message) => {
|
||||||
|
const socketsToSend = await findClientsForStreamer(channel)
|
||||||
|
for(let socket of socketsToSend) {
|
||||||
|
let payload = {
|
||||||
|
username,
|
||||||
|
type,
|
||||||
|
channel,
|
||||||
|
message
|
||||||
|
}
|
||||||
|
socket.send(JSON.stringify(payload))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
export default proxyRouter
|
103
tsconfig.json
Normal file
103
tsconfig.json
Normal file
|
@ -0,0 +1,103 @@
|
||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
/* Visit https://aka.ms/tsconfig to read more about this file */
|
||||||
|
|
||||||
|
/* Projects */
|
||||||
|
// "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
|
||||||
|
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
|
||||||
|
// "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */
|
||||||
|
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */
|
||||||
|
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
|
||||||
|
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
|
||||||
|
|
||||||
|
/* Language and Environment */
|
||||||
|
"target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
|
||||||
|
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
|
||||||
|
// "jsx": "preserve", /* Specify what JSX code is generated. */
|
||||||
|
// "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */
|
||||||
|
// "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */
|
||||||
|
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
|
||||||
|
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
|
||||||
|
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
|
||||||
|
// "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
|
||||||
|
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
|
||||||
|
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
|
||||||
|
// "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */
|
||||||
|
|
||||||
|
/* Modules */
|
||||||
|
"module": "commonjs", /* Specify what module code is generated. */
|
||||||
|
// "rootDir": "./", /* Specify the root folder within your source files. */
|
||||||
|
// "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */
|
||||||
|
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
|
||||||
|
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
|
||||||
|
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
|
||||||
|
// "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */
|
||||||
|
// "types": [], /* Specify type package names to be included without being referenced in a source file. */
|
||||||
|
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
|
||||||
|
// "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */
|
||||||
|
// "resolveJsonModule": true, /* Enable importing .json files. */
|
||||||
|
// "noResolve": true, /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
|
||||||
|
|
||||||
|
/* JavaScript Support */
|
||||||
|
// "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
|
||||||
|
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
|
||||||
|
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
|
||||||
|
|
||||||
|
/* Emit */
|
||||||
|
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
|
||||||
|
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
|
||||||
|
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
|
||||||
|
// "sourceMap": true, /* Create source map files for emitted JavaScript files. */
|
||||||
|
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
|
||||||
|
// "outDir": "./", /* Specify an output folder for all emitted files. */
|
||||||
|
// "removeComments": true, /* Disable emitting comments. */
|
||||||
|
// "noEmit": true, /* Disable emitting files from a compilation. */
|
||||||
|
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
|
||||||
|
// "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */
|
||||||
|
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
|
||||||
|
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
|
||||||
|
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
|
||||||
|
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
|
||||||
|
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
|
||||||
|
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
|
||||||
|
// "newLine": "crlf", /* Set the newline character for emitting files. */
|
||||||
|
// "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
|
||||||
|
// "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */
|
||||||
|
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
|
||||||
|
// "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */
|
||||||
|
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
|
||||||
|
// "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
|
||||||
|
|
||||||
|
/* Interop Constraints */
|
||||||
|
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
|
||||||
|
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
|
||||||
|
"esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
|
||||||
|
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
|
||||||
|
"forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */
|
||||||
|
|
||||||
|
/* Type Checking */
|
||||||
|
"strict": true, /* Enable all strict type-checking options. */
|
||||||
|
// "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */
|
||||||
|
// "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */
|
||||||
|
// "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
|
||||||
|
// "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
|
||||||
|
// "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */
|
||||||
|
// "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */
|
||||||
|
// "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */
|
||||||
|
// "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */
|
||||||
|
// "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */
|
||||||
|
// "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */
|
||||||
|
// "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
|
||||||
|
// "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
|
||||||
|
// "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
|
||||||
|
// "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */
|
||||||
|
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
|
||||||
|
// "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */
|
||||||
|
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
|
||||||
|
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
|
||||||
|
|
||||||
|
/* Completeness */
|
||||||
|
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
|
||||||
|
"skipLibCheck": true /* Skip type checking all .d.ts files. */
|
||||||
|
}
|
||||||
|
}
|
3
types/looseTypes.ts
Normal file
3
types/looseTypes.ts
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
export interface LooseObject {
|
||||||
|
[key: string]: any
|
||||||
|
}
|
17
types/scraping/Chat.ts
Normal file
17
types/scraping/Chat.ts
Normal file
|
@ -0,0 +1,17 @@
|
||||||
|
export interface TwitchChatOptions {
|
||||||
|
login: {
|
||||||
|
username: string,
|
||||||
|
password: string
|
||||||
|
},
|
||||||
|
channels: string[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export const MessageTypes = ['PRIVMSG', 'WHISPER']
|
||||||
|
export type MessageType = typeof MessageTypes[number];
|
||||||
|
|
||||||
|
export interface Metadata {
|
||||||
|
username: string
|
||||||
|
messageType: MessageType
|
||||||
|
channel: string
|
||||||
|
message: string
|
||||||
|
}
|
28
types/scraping/Streamer.ts
Normal file
28
types/scraping/Streamer.ts
Normal file
|
@ -0,0 +1,28 @@
|
||||||
|
export interface Social {
|
||||||
|
type: string | null
|
||||||
|
text: string,
|
||||||
|
link: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StreamData {
|
||||||
|
tags: string[]
|
||||||
|
title: string
|
||||||
|
topic: string
|
||||||
|
startedAt: number
|
||||||
|
viewers: number
|
||||||
|
preview: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StreamerData {
|
||||||
|
username: string
|
||||||
|
followers: number
|
||||||
|
followersAbbv: string
|
||||||
|
isLive: boolean
|
||||||
|
about: string
|
||||||
|
socials?: Social[]
|
||||||
|
pfp: string
|
||||||
|
stream: StreamData | null
|
||||||
|
isPartner: boolean
|
||||||
|
colorHex: string
|
||||||
|
id: number
|
||||||
|
}
|
46
util/logger.ts
Normal file
46
util/logger.ts
Normal file
|
@ -0,0 +1,46 @@
|
||||||
|
import { randomUUID } from 'crypto'
|
||||||
|
import { createLogger, format, transports } from 'winston'
|
||||||
|
import { NextFunction, Request, Response } from 'express'
|
||||||
|
|
||||||
|
const logLevels = {
|
||||||
|
fatal: 0,
|
||||||
|
error: 1,
|
||||||
|
warn: 2,
|
||||||
|
info: 3,
|
||||||
|
debug: 4,
|
||||||
|
trace: 5,
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
export const logger = createLogger({
|
||||||
|
format: format.combine(format.timestamp(), format.json()),
|
||||||
|
transports: [
|
||||||
|
new transports.Console({
|
||||||
|
format: format.combine(
|
||||||
|
format.colorize(),
|
||||||
|
format.simple()
|
||||||
|
)
|
||||||
|
}),
|
||||||
|
new transports.File({ filename: './serverLog.log' })
|
||||||
|
],
|
||||||
|
levels: logLevels
|
||||||
|
});
|
||||||
|
|
||||||
|
export const errorHandler = (err: Error, req: Request, res: Response, next: NextFunction) => {
|
||||||
|
if (res.headersSent) {
|
||||||
|
return next(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
res.status(500).send({ status: 'error', message: err.message, code: res.locals.uuid })
|
||||||
|
logger.warn({
|
||||||
|
message: err.message,
|
||||||
|
endpoint: req.originalUrl,
|
||||||
|
reqId: res.locals.uuid,
|
||||||
|
origin: req.headers.origin,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export const uuid = (req: Request, res: Response, next: NextFunction) => {
|
||||||
|
res.locals.uuid = randomUUID()
|
||||||
|
next()
|
||||||
|
}
|
92
util/scraping/chat/chat.ts
Normal file
92
util/scraping/chat/chat.ts
Normal file
|
@ -0,0 +1,92 @@
|
||||||
|
import { EventEmitter } from 'stream';
|
||||||
|
import WebSocket from 'ws'
|
||||||
|
import { TwitchChatOptions, Metadata, MessageType, MessageTypes } from '../../../types/scraping/Chat'
|
||||||
|
import { parseUsername } from './utils';
|
||||||
|
import { logger } from '../../logger';
|
||||||
|
|
||||||
|
export declare interface TwitchChat {
|
||||||
|
on(event: 'PRIVMSG', listener: (username: string, messageType: MessageType, channel: string, message: string) => void): this
|
||||||
|
}
|
||||||
|
|
||||||
|
export class TwitchChat extends EventEmitter{
|
||||||
|
public channels: string[]
|
||||||
|
private url = 'wss://irc-ws.chat.twitch.tv:443'
|
||||||
|
private ws: WebSocket | null;
|
||||||
|
private isConnected: boolean = false
|
||||||
|
private manualDisconnect: boolean = false
|
||||||
|
|
||||||
|
constructor(options: TwitchChatOptions) {
|
||||||
|
super()
|
||||||
|
this.channels = options.channels
|
||||||
|
this.ws = null
|
||||||
|
}
|
||||||
|
|
||||||
|
private parser() {
|
||||||
|
this.ws?.on('message', (data) => {
|
||||||
|
console.log(this.channels)
|
||||||
|
let normalData = data.toString()
|
||||||
|
let splitted = normalData.split(":")
|
||||||
|
|
||||||
|
let metadata = splitted[1].split(' ')
|
||||||
|
let message = splitted[2]
|
||||||
|
|
||||||
|
if(!MessageTypes.includes(metadata[1])) return;
|
||||||
|
|
||||||
|
let parsedMetadata: Metadata = {
|
||||||
|
username: parseUsername(metadata[0]),
|
||||||
|
messageType: metadata[1],
|
||||||
|
channel: metadata[2].replace('#', ''),
|
||||||
|
message: message
|
||||||
|
}
|
||||||
|
|
||||||
|
this.createEmit(parsedMetadata)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
private createEmit(data: Metadata) {
|
||||||
|
this.emit(data.messageType, ...Object.values(data))
|
||||||
|
}
|
||||||
|
|
||||||
|
public async connect() {
|
||||||
|
console.log('ss')
|
||||||
|
this.ws = new WebSocket(this.url)
|
||||||
|
this.isConnected = true
|
||||||
|
|
||||||
|
this.ws.onclose = () => {
|
||||||
|
logger.info('Disconnected from twitch IRC'),
|
||||||
|
logger.info(`Subscribed to channels ${this.channels}`)
|
||||||
|
if(this.manualDisconnect) return
|
||||||
|
const toEmit = {
|
||||||
|
type: 'SERVERMSG',
|
||||||
|
message: 'Disconnected'
|
||||||
|
}
|
||||||
|
this.emit(JSON.stringify(toEmit))
|
||||||
|
|
||||||
|
this.ws = null
|
||||||
|
this.isConnected = false
|
||||||
|
this.connect()
|
||||||
|
}
|
||||||
|
|
||||||
|
this.ws.onopen = () => {
|
||||||
|
if(this.ws) {
|
||||||
|
this.ws.send('PASS none')
|
||||||
|
this.ws.send('NICK justinfan333333333333')
|
||||||
|
|
||||||
|
for(let channel of this.channels) {
|
||||||
|
this.ws.send(`JOIN #${channel}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
this.parser()
|
||||||
|
return Promise.resolve()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public addStreamer(streamerName: string) {
|
||||||
|
if(!this.isConnected) return;
|
||||||
|
this.channels.push(streamerName)
|
||||||
|
this.ws!.send(`JOIN #${streamerName}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
4
util/scraping/chat/utils.ts
Normal file
4
util/scraping/chat/utils.ts
Normal file
|
@ -0,0 +1,4 @@
|
||||||
|
export const parseUsername = (rawUsername: String) => {
|
||||||
|
const splitted = rawUsername.split('!')
|
||||||
|
return splitted[0]
|
||||||
|
}
|
419
util/scraping/extractor/index.ts
Normal file
419
util/scraping/extractor/index.ts
Normal file
|
@ -0,0 +1,419 @@
|
||||||
|
import { LooseObject } from "../../../types/looseTypes"
|
||||||
|
import { StreamerData, StreamData, Social } from "../../../types/scraping/Streamer"
|
||||||
|
|
||||||
|
const base64 = (data: String) => {
|
||||||
|
return Buffer.from(data).toString('base64url')
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Class that interacts with the Twitch api
|
||||||
|
*/
|
||||||
|
export class TwitchAPI {
|
||||||
|
public readonly twitchUrl = 'https://gql.twitch.tv/gql'
|
||||||
|
public headers = {
|
||||||
|
"Client-Id": "kimne78kx3ncx6brgo4mv6wki5h1ko"
|
||||||
|
}
|
||||||
|
|
||||||
|
constructor() {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets information about a streamer, like socials, about, and more.
|
||||||
|
* @see StreamerData
|
||||||
|
* @param streamerName The username of the streamer
|
||||||
|
* @returns Promise<StreamerData>
|
||||||
|
*/
|
||||||
|
public getStreamerInfo = async (streamerName: string) => {
|
||||||
|
const payload = [
|
||||||
|
{
|
||||||
|
"operationName": "ChannelRoot_AboutPanel",
|
||||||
|
"variables": {
|
||||||
|
"channelLogin": streamerName,
|
||||||
|
"skipSchedule": false
|
||||||
|
},
|
||||||
|
"extensions": {
|
||||||
|
"persistedQuery": {
|
||||||
|
"version": 1,
|
||||||
|
"sha256Hash": "6089531acef6c09ece01b440c41978f4c8dc60cb4fa0124c9a9d3f896709b6c6"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"operationName":"StreamMetadata",
|
||||||
|
"variables":{
|
||||||
|
"channelLogin": streamerName
|
||||||
|
},
|
||||||
|
"extensions":{
|
||||||
|
"persistedQuery":{
|
||||||
|
"version":1,
|
||||||
|
"sha256Hash":"a647c2a13599e5991e175155f798ca7f1ecddde73f7f341f39009c14dbf59962"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"operationName": "StreamTagsTrackingChannel",
|
||||||
|
"variables": {
|
||||||
|
"channel": streamerName
|
||||||
|
},
|
||||||
|
"extensions": {
|
||||||
|
"persistedQuery": {
|
||||||
|
"version": 1,
|
||||||
|
"sha256Hash": "6aa3851aaaf88c320d514eb173563d430b28ed70fdaaf7eeef6ed4b812f48608"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"operationName": "VideoPreviewOverlay",
|
||||||
|
"variables": {
|
||||||
|
"login": streamerName
|
||||||
|
},
|
||||||
|
"extensions": {
|
||||||
|
"persistedQuery": {
|
||||||
|
"version": 1,
|
||||||
|
"sha256Hash": "9515480dee68a77e667cb19de634739d33f243572b007e98e67184b1a5d8369f"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"operationName": "UseViewCount",
|
||||||
|
"variables": {
|
||||||
|
"channelLogin": streamerName
|
||||||
|
},
|
||||||
|
"extensions": {
|
||||||
|
"persistedQuery": {
|
||||||
|
"version": 1,
|
||||||
|
"sha256Hash": "00b11c9c428f79ae228f30080a06ffd8226a1f068d6f52fbc057cbde66e994c2"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
const res = await fetch(this.twitchUrl, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
headers: this.headers
|
||||||
|
})
|
||||||
|
|
||||||
|
const data = await res.json()
|
||||||
|
const rawStreamerData = data[0].data
|
||||||
|
|
||||||
|
|
||||||
|
// get socials
|
||||||
|
const socials: LooseObject[] = []
|
||||||
|
if (rawStreamerData.user.channel && rawStreamerData.user.channel.socialMedias) {
|
||||||
|
for (let social of rawStreamerData.user.channel.socialMedias) {
|
||||||
|
socials.push({
|
||||||
|
type: social.name,
|
||||||
|
name: social.title,
|
||||||
|
link: social.url
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// check if is liver
|
||||||
|
const rawStreamData = data[1].data.user.stream
|
||||||
|
let parsedStream: StreamData | null;
|
||||||
|
if(!rawStreamData) {
|
||||||
|
parsedStream = null
|
||||||
|
} else {
|
||||||
|
const tags: string[] = []
|
||||||
|
for (let tagData of data[2].data.user.stream.freeformTags) {
|
||||||
|
tags.push(tagData.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
const previewUrl = `${process.env.URL}/proxy/img/${base64(data[3].data.user.stream.previewImageURL)}`
|
||||||
|
|
||||||
|
parsedStream = {
|
||||||
|
title: data[1].data.user.lastBroadcast.title,
|
||||||
|
topic: rawStreamData.game.name,
|
||||||
|
startedAt: new Date(rawStreamData.createdAt).valueOf(),
|
||||||
|
tags,
|
||||||
|
viewers: Number(data[4].data.user.stream.viewersCount),
|
||||||
|
preview: data[3].data.user.stream.previewImageURL
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const abbreviatedFollowers = Intl.NumberFormat('en-US', {
|
||||||
|
notation: "compact",
|
||||||
|
maximumFractionDigits: 1
|
||||||
|
}).format(rawStreamerData.user.followers.totalCount)
|
||||||
|
|
||||||
|
const streamerData: StreamerData = {
|
||||||
|
username: rawStreamerData.user.displayName,
|
||||||
|
about: rawStreamerData.user.description,
|
||||||
|
pfp: rawStreamerData.user.profileImageURL,
|
||||||
|
followers: rawStreamerData.user.followers.totalCount,
|
||||||
|
socials: socials as Social[],
|
||||||
|
isLive: (!!parsedStream),
|
||||||
|
isPartner: rawStreamerData.user.isPartner,
|
||||||
|
followersAbbv: abbreviatedFollowers,
|
||||||
|
colorHex: '#' + rawStreamerData.user.primaryColorHex,
|
||||||
|
id: Number(rawStreamerData.user.id),
|
||||||
|
stream: parsedStream
|
||||||
|
}
|
||||||
|
|
||||||
|
return Promise.resolve(streamerData)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the current viewers of a stream
|
||||||
|
* @param streamerName The username of the streamer
|
||||||
|
* @returns Promise<number>
|
||||||
|
*/
|
||||||
|
public getViewers = async (streamerName: string) => {
|
||||||
|
const payload = [
|
||||||
|
{
|
||||||
|
"operationName": "UseViewCount",
|
||||||
|
"variables": {
|
||||||
|
"channelLogin": streamerName
|
||||||
|
},
|
||||||
|
"extensions": {
|
||||||
|
"persistedQuery": {
|
||||||
|
"version": 1,
|
||||||
|
"sha256Hash": "00b11c9c428f79ae228f30080a06ffd8226a1f068d6f52fbc057cbde66e994c2"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
const res = await fetch(this.twitchUrl, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
headers: this.headers
|
||||||
|
})
|
||||||
|
|
||||||
|
const rawData = await res.json()
|
||||||
|
|
||||||
|
if(!rawData[0].data.user.stream)
|
||||||
|
return Promise.reject(new Error(`Streamer ${streamerName} is not live`))
|
||||||
|
|
||||||
|
return Promise.resolve(rawData[0].data.user.stream.viewersCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
public isLive = async (streamerName: string) => {
|
||||||
|
const payload = [
|
||||||
|
{
|
||||||
|
"operationName": "UseViewCount",
|
||||||
|
"variables": {
|
||||||
|
"channelLogin": streamerName
|
||||||
|
},
|
||||||
|
"extensions": {
|
||||||
|
"persistedQuery": {
|
||||||
|
"version": 1,
|
||||||
|
"sha256Hash": "00b11c9c428f79ae228f30080a06ffd8226a1f068d6f52fbc057cbde66e994c2"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
const res = await fetch(this.twitchUrl, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
headers: this.headers
|
||||||
|
})
|
||||||
|
|
||||||
|
const rawData = await res.json()
|
||||||
|
|
||||||
|
if(!rawData[0].data.user.stream)
|
||||||
|
return Promise.resolve(false)
|
||||||
|
|
||||||
|
return Promise.resolve(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the stream playlist file from twitch
|
||||||
|
* @param streamerName The username of the streamer
|
||||||
|
* @returns Promise<boolean>
|
||||||
|
*/
|
||||||
|
public getStream = async (streamerName: string) => {
|
||||||
|
const isLive = await this.isLive(streamerName)
|
||||||
|
if(!isLive) return Promise.reject(new Error(`Streamer ${streamerName} is not live`))
|
||||||
|
|
||||||
|
// Get token
|
||||||
|
const payload = {
|
||||||
|
"operationName": "PlaybackAccessToken_Template",
|
||||||
|
"query": "query PlaybackAccessToken_Template($login: String!, $isLive: Boolean!, $vodID: ID!, $isVod: Boolean!, $playerType: String!) { streamPlaybackAccessToken(channelName: $login, params: {platform: \"web\", playerBackend: \"mediaplayer\", playerType: $playerType}) @include(if: $isLive) { value signature __typename } videoPlaybackAccessToken(id: $vodID, params: {platform: \"web\", playerBackend: \"mediaplayer\", playerType: $playerType}) @include(if: $isVod) { value signature __typename }}",
|
||||||
|
"variables": {
|
||||||
|
"isLive": true,
|
||||||
|
"login": streamerName,
|
||||||
|
"isVod": false,
|
||||||
|
"vodID": "",
|
||||||
|
"playerType": "site"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var res = await fetch(this.twitchUrl, {
|
||||||
|
headers: this.headers,
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
method: 'POST'
|
||||||
|
})
|
||||||
|
|
||||||
|
const data = await res.json()
|
||||||
|
|
||||||
|
const token = data.data.streamPlaybackAccessToken.value
|
||||||
|
const signature = data.data.streamPlaybackAccessToken.signature
|
||||||
|
|
||||||
|
const playlistUrl = `https://usher.ttvnw.net/api/channel/hls/${streamerName.toLowerCase()}.m3u8`
|
||||||
|
const params = `?sig=${signature}&token=${token}`
|
||||||
|
var res = await fetch(playlistUrl + params, {
|
||||||
|
headers: this.headers
|
||||||
|
})
|
||||||
|
|
||||||
|
return await res.text()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the homepage discovery tab of twitch
|
||||||
|
* @param limit Maximum categories to get
|
||||||
|
* @param cursor The current page you're at (for pagination)
|
||||||
|
|
||||||
|
*/
|
||||||
|
public getDirectory = async (limit: number, cursor?: number) => {
|
||||||
|
const payload: any[] = [
|
||||||
|
{
|
||||||
|
"operationName": "BrowsePage_AllDirectories",
|
||||||
|
"variables": {
|
||||||
|
"limit": limit,
|
||||||
|
"options": {
|
||||||
|
"recommendationsContext": {
|
||||||
|
"platform": "web"
|
||||||
|
},
|
||||||
|
"requestID": "JIRA-VXP-2397",
|
||||||
|
"sort": "RELEVANCE",
|
||||||
|
"tags": []
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"extensions": {
|
||||||
|
"persistedQuery": {
|
||||||
|
"version": 1,
|
||||||
|
"sha256Hash": "1d1914ca3cbfaa607ecd5595b2e305e96acf987c8f25328f7713b25f604c4668"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
if(cursor)
|
||||||
|
payload[0].variables.cursor = cursor
|
||||||
|
|
||||||
|
const res = await fetch(this.twitchUrl, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
headers: this.headers
|
||||||
|
})
|
||||||
|
const data = await res.json()
|
||||||
|
const categories = data[0].data.directoriesWithTags.edges
|
||||||
|
let formattedCategories = []
|
||||||
|
|
||||||
|
for (let category of categories) {
|
||||||
|
let tags = []
|
||||||
|
for (let tag of category.node.tags) {
|
||||||
|
tags.push(tag.tagName)
|
||||||
|
}
|
||||||
|
|
||||||
|
formattedCategories.push({
|
||||||
|
name: category.node.name,
|
||||||
|
displayName: category.node.displayName,
|
||||||
|
viewers: category.node.viewersCount,
|
||||||
|
tags: tags,
|
||||||
|
createdAt: category.node.originalReleaseDate,
|
||||||
|
image: `${process.env.URL}/proxy/img/${base64(category.node.avatarURL)}`
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return formattedCategories
|
||||||
|
}
|
||||||
|
|
||||||
|
public getDirectoryGame = async (name: string, streamLimit: number) => {
|
||||||
|
const payload: any[] = [
|
||||||
|
{
|
||||||
|
"operationName": "DirectoryPage_Game",
|
||||||
|
"variables": {
|
||||||
|
"imageWidth": 50,
|
||||||
|
"name": name,
|
||||||
|
"options": {
|
||||||
|
"sort": "RELEVANCE",
|
||||||
|
"recommendationsContext": {
|
||||||
|
"platform": "web"
|
||||||
|
},
|
||||||
|
"requestID": "JIRA-VXP-2397",
|
||||||
|
"freeformTags": null,
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
|
"sortTypeIsRecency": false,
|
||||||
|
"limit": streamLimit
|
||||||
|
},
|
||||||
|
"extensions": {
|
||||||
|
"persistedQuery": {
|
||||||
|
"version": 1,
|
||||||
|
"sha256Hash": "df4bb6cc45055237bfaf3ead608bbafb79815c7100b6ee126719fac3762ddf8b"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"operationName": "Directory_DirectoryBanner",
|
||||||
|
"variables": {
|
||||||
|
"name": name
|
||||||
|
},
|
||||||
|
"extensions": {
|
||||||
|
"persistedQuery": {
|
||||||
|
"version": 1,
|
||||||
|
"sha256Hash": "2670fbecd8fbea0211c56528d6eff5752ef9d6c73cd5238d395784b46335ded4"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
const res = await fetch(this.twitchUrl, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
headers: this.headers
|
||||||
|
})
|
||||||
|
const data = await res.json()
|
||||||
|
|
||||||
|
if(!data[0].data.game) return null;
|
||||||
|
|
||||||
|
|
||||||
|
let streams = []
|
||||||
|
if(data[0].data.game.streams)
|
||||||
|
streams = data[0].data.game.streams.edges
|
||||||
|
let formatedStreams = []
|
||||||
|
|
||||||
|
for(let stream of streams) {
|
||||||
|
let tags = []
|
||||||
|
for (let tag of stream.node.freeformTags) {
|
||||||
|
tags.push(tag.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
formatedStreams.push({
|
||||||
|
title: stream.node.title,
|
||||||
|
viewers: stream.node.viewersCount,
|
||||||
|
preview: stream.node.previewImageURL,
|
||||||
|
tags,
|
||||||
|
streamer: {
|
||||||
|
name: stream.node.broadcaster.displayName,
|
||||||
|
pfp: stream.node.broadcaster.profileImageURL,
|
||||||
|
colorHex: stream.node.broadcaster.primaryColorHex
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const rawGameData = data[1].data.game;
|
||||||
|
let tags = []
|
||||||
|
for(let tag of rawGameData.tags) {
|
||||||
|
tags.push(tag.tagName)
|
||||||
|
}
|
||||||
|
|
||||||
|
const formatedGameData = {
|
||||||
|
name: rawGameData.name,
|
||||||
|
cover: rawGameData.avatarURL,
|
||||||
|
description: rawGameData.description,
|
||||||
|
viewers: rawGameData.viewersCount,
|
||||||
|
followers: rawGameData.followersCount,
|
||||||
|
tags,
|
||||||
|
streams: formatedStreams
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
return formatedGameData
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in a new issue