safetwitch/server/routes/profile/profileRoute.ts

import { Router } from 'express'
import puppeteer from 'puppeteer-extra'
import { Browser, Page } from 'puppeteer'
import { PuppeteerExtraPluginAdblocker } from 'puppeteer-extra-plugin-adblocker'
import { LooseObject } from '../../types/looseTypes'
import { Streamlink } from '@dragongoose/streamlink'

puppeteer.use(new PuppeteerExtraPluginAdblocker({ 
    blockTrackersAndAnnoyances: true 
}))

const profileRouter = Router()

export interface Socials {
    type: string | null
    text: string,
    link: string
}

export interface StreamData {
    tags: string[]
    title: string
    topic: string
    startedAt: number
    qualities: string[]
}

export interface StreamerData {
    username: string,
    followers: number,
    followersAbbv: string,
    isLive: boolean,
    about: string,
    socials?: string[],
    pfp: string;
    stream?: StreamData
}


const abbreviatedNumberToNumber = (num: string) => {
    const base = parseFloat(num)
    
    const matches: {[k: string]: number} = {
        'k': 1000,
        'm': 1000000,
        'b': 1000000000
    }

    const abbreviation: string = num.charAt(num.length - 1).toLowerCase()


    if(matches[abbreviation]) {
        const numberOnly: number = Number(num.slice(0, -1))
        return numberOnly * matches[abbreviation]
    } else {
        return null
    }
}

// https:// advancedweb.hu/how-to-speed-up-puppeteer-scraping-with-parallelization/
const withBrowser = async (fn: Function) => {
	const browser = await puppeteer.launch({
        headless: true,
        args: ['--no-sandbox']
    });
	try {
		return await fn(browser);
	} finally {
		await browser.close();
	}
}

const withPage = (browser: Browser) => async (fn: Function) => {
	const page = await browser.newPage();
    //await page.tracing.start({ path: '../profile.json', screenshots: true });
    try {
        return await fn(page);
    } finally {
        //await page.tracing.stop();
        await page.close();
    }
}

let isLive: boolean

const getStreamData = async (page: Page) => {
    const streamData: LooseObject = {}

    if(!isLive) return null

    // Get stream tags
    const tagsSelector = '.eUxEWt * span'
    const tags: string[] = await page.$$eval(tagsSelector, elements => elements.map(el => el.innerHTML))
    streamData.tags = tags

    // Get stream title
    const titleSelector = 'h2.CoreText-sc-1txzju1-0'
    const title: string = await page.$eval(titleSelector, element => element.innerText)
    streamData.title = title

    // Get topic
    const topicSelector = '.hfMGmo'
    const topic = await page.$eval(topicSelector, element => element.textContent)
    streamData.topic = topic

    // Get Start time
    const liveTimeSelector = '.live-time'

    // formated as HH:MM:SS
    const liveTime = await page.$eval(liveTimeSelector, element => element.textContent)
    if(!liveTime) return
    const liveTimeSplit: number[] = liveTime.split(':').map(Number)
    let date = new Date()
    let { hours, minutes, seconds } = { hours: date.getHours(), minutes: date.getMinutes(), seconds: date.getSeconds()}
    
    // Subtracts current live time from current
    // date to get the time the stream started
    date.setHours(hours - liveTimeSplit[0])
    date.setMinutes(minutes - liveTimeSplit[1])
    date.setSeconds(seconds - liveTimeSplit[2])

    streamData.startedAt = date.getTime()

    return streamData as StreamData
}

const getAboutData = async (page: Page) => {
        const aboutData: LooseObject = {}

        if (!isLive) {
            // Get data from about page
            const aboutPageButtonSelector = 'li.InjectLayout-sc-1i43xsx-0:nth-child(2) > a:nth-child(1) > div:nth-child(1) > div:nth-child(1) > p:nth-child(1)'
            await page.click(aboutPageButtonSelector)
        }
        await page.waitForSelector('.ccXeNc')
    
        const followersSelector = '.kuAEke'
        const followers = await page.$eval(followersSelector, element => element.innerHTML)
        aboutData.followersAbbv = followers
        aboutData.followers = abbreviatedNumberToNumber(followers)
    
        const aboutSectionSelector = '.kLFSJC'
        const aboutSection = await page.$eval(aboutSectionSelector, element => element.innerHTML)
        aboutData.about = aboutSection
    
        const socialSelector = '.ccXeNc * a'
        const socials: Socials[] = await page.$$eval(socialSelector, elements => elements.map((el) => {
            
            const getHostName = (url: string) => {
                const match = url.match(/:\/\/(www[0-9]?\.)?(.[^/:]+)/i);
                if (match != null && match.length > 2 && typeof match[2] === 'string' && match[2].length > 0) {
                    const hostname = match[2].split(".");
                    return hostname[0];
                }
                else {
                    return null;
                }
            }

            const validHosts = ['instagram', 'youtube', 'discord', 'tiktok','twitter']
            const socialHost = getHostName(el.href) || el.href || ''
            let type: string | null = socialHost
            if(!validHosts.includes(socialHost)) 
                type = null

            return {
                type,
                link: el.href,
                text: el.innerText
            }
        }))
        aboutData.socials = socials
    
        const profilePictureSelector = 'figure.ScAvatar-sc-144b42z-0:nth-child(2) > img:nth-child(1)'
        const profilePicutre = await page.$eval(profilePictureSelector, element => element.getAttribute('src'))
        aboutData.pfp = profilePicutre

        return aboutData as StreamerData
}

const getStreamerData = async (username: string): Promise<StreamerData> => {
    let recoveredData: LooseObject = {}

    await withBrowser(async (browser: Browser) => {
        const result = await withPage(browser)(async (page: Page) => {
            await page.goto(`https://twitch.tv/${username}`)    

            return Promise.all([getStreamData(page), getAboutData(page)])
        })
        
        recoveredData = result[1]
        recoveredData.stream = result[0]
        if(result[0] !== null) recoveredData.isLive = true

        await browser.close()
    })

    recoveredData.username = username
    return recoveredData as StreamerData
}

profileRouter.get('/users/:username', async (req, res, next) => {
    const username = req.params.username
    const streamlink = new Streamlink(`https://twitch.tv/${username}`, {})
    isLive = await streamlink.isLive()

    let streamerData = await getStreamerData(username)
        .catch(next)

    if(streamerData && streamerData.stream && isLive)
        streamerData.stream.qualities = await streamlink.getQualities()
    
    if(streamerData) {
        streamerData.isLive = isLive
        res.send(streamerData)
    }
})

export default profileRouter