/', + $data, + $keys + ); + + print_r($keys); + + if(count($keys) === 0){ + + $this->do404("Bing has not archived this URL."); + } + + $keys = explode("|", $keys[1]); + $count = count($keys); + + //header("Location: https://cc.bingj.com/cache.aspx?d=" . $keys[$count - 2] . "&w=" . $keys[$count - 1]); + echo("Location: https://cc.bingj.com/cache.aspx?d=" . $keys[$count - 2] . "&w=" . $keys[$count - 1]); + } + + public function do404($text){ + + include "lib/frontend.php"; + $frontend = new frontend(); + + echo + $frontend->load( + "error.html", + [ + "title" => "Shit", + "text" => $text + ] + ); + + die(); + } + + public function validate_url($url){ + + $url_parts = parse_url($url); + + // check if required parts are there + if( + !isset($url_parts["scheme"]) || + !( + $url_parts["scheme"] == "http" || + $url_parts["scheme"] == "https" + ) || + !isset($url_parts["host"]) + ){ + return false; + } + + if( + // if its not an RFC-valid URL + !filter_var($url, FILTER_VALIDATE_URL) + ){ + return false; + } + + $ip = + str_replace( + ["[", "]"], // handle ipv6 + "", + $url_parts["host"] + ); + + // if its not an IP + if(!filter_var($ip, FILTER_VALIDATE_IP)){ + + // resolve domain's IP + $ip = gethostbyname($url_parts["host"] . "."); + } + + // check if its localhost + return filter_var( + $ip, + FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE + ); + } +} diff --git a/lib/classic.png b/lib/classic.png new file mode 100644 index 0000000..3d2b8fc Binary files /dev/null and b/lib/classic.png differ diff --git a/lib/curlproxy.php b/lib/curlproxy.php new file mode 100644 index 0000000..846fbb7 --- /dev/null +++ b/lib/curlproxy.php @@ -0,0 +1,652 @@ +cache = $cache; + } + + public function do404(){ + + http_response_code(404); + header("Content-Type: image/png"); + + $handle = fopen("lib/img404.png", "r"); + echo fread($handle, filesize("lib/img404.png")); + fclose($handle); + + die(); + return; + } + + public function getabsoluteurl($path, $relative){ + + if($this->validateurl($path)){ + + return $path; + } + + if(substr($path, 0, 2) == "//"){ + + return "https:" . $path; + } + + $url = null; + + $relative = parse_url($relative); + $url = $relative["scheme"] . "://"; + + if( + isset($relative["user"]) && + isset($relative["pass"]) + ){ + + $url .= $relative["user"] . ":" . $relative["pass"] . "@"; + } + + $url .= $relative["host"]; + + if(isset($relative["path"])){ + + $relative["path"] = explode( + "/", + $relative["path"] + ); + + unset($relative["path"][count($relative["path"]) - 1]); + $relative["path"] = implode("/", $relative["path"]); + + $url .= $relative["path"]; + } + + if( + strlen($path) !== 0 && + $path[0] !== "/" + ){ + + $url .= "/"; + } + + $url .= $path; + + return $url; + } + + public function validateurl($url){ + + $url_parts = parse_url($url); + + // check if required parts are there + if( + !isset($url_parts["scheme"]) || + !( + $url_parts["scheme"] == "http" || + $url_parts["scheme"] == "https" + ) || + !isset($url_parts["host"]) + ){ + return false; + } + + $ip = + str_replace( + ["[", "]"], // handle ipv6 + "", + $url_parts["host"] + ); + + // if its not an IP + if(!filter_var($ip, FILTER_VALIDATE_IP)){ + + // resolve domain's IP + $ip = gethostbyname($url_parts["host"] . "."); + } + + // check if its localhost + if( + filter_var( + $ip, + FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE + ) === false + ){ + + return false; + } + + return true; + } + + public function get($url, $reqtype = self::req_web, $acceptallcodes = false, $referer = null, $redirectcount = 0){ + + if($redirectcount === 5){ + + throw new Exception("Too many redirects"); + } + + // sanitize URL + try{ + + $this->validateurl($url); + }catch(Exception $error){ + + throw new Exception($error->getMessage()); + } + + $this->clientcache(); + + $curl = curl_init(); + + curl_setopt($curl, CURLOPT_URL, $url); + curl_setopt($curl, CURLOPT_ENCODING, ""); // default encoding + curl_setopt($curl, CURLOPT_HEADER, 1); + + switch($reqtype){ + case self::req_web: + curl_setopt( + $curl, + CURLOPT_HTTPHEADER, + [ + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate", + "DNT: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1" + ] + ); + break; + + case self::req_image: + + if($referer === null){ + $referer = explode("/", $url, 4); + array_pop($referer); + + $referer = implode("/", $referer); + } + + curl_setopt( + $curl, + CURLOPT_HTTPHEADER, + [ + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/107.0", + "Accept: image/avif,image/webp,*/*", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate", + "DNT: 1", + "Connection: keep-alive", + "Referer: {$referer}" + ] + ); + break; + } + + curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curl, CURLOPT_TIMEOUT, 30); + + // limit size of payloads + curl_setopt($curl, CURLOPT_BUFFERSIZE, 1024); + curl_setopt($curl, CURLOPT_NOPROGRESS, false); + curl_setopt( + $curl, + CURLOPT_PROGRESSFUNCTION, + function($downloadsize, $downloaded, $uploadsize, $uploaded + ){ + + // if $downloaded exceeds 100MB, fuck off + return ($downloaded > 100000000) ? 1 : 0; + }); + + $body = curl_exec($curl); + + if(curl_errno($curl)){ + + throw new Exception(curl_error($curl)); + } + + curl_close($curl); + + $headers = []; + $http = null; + + while(true){ + + $header = explode("\n", $body, 2); + $body = $header[1]; + + if($http === null){ + + // http/1.1 200 ok + $header = explode("/", $header[0], 2); + $header = explode(" ", $header[1], 3); + + $http = [ + "version" => (float)$header[0], + "code" => (int)$header[1] + ]; + + continue; + } + + if(trim($header[0]) == ""){ + + // reached end of headers + break; + } + + $header = explode(":", $header[0], 2); + + // malformed headers + if(count($header) !== 2){ continue; } + + $headers[strtolower(trim($header[0]))] = trim($header[1]); + } + + // check http code + if( + $http["code"] >= 300 && + $http["code"] <= 309 + ){ + + // redirect + if(!isset($headers["location"])){ + + throw new Exception("Broken redirect"); + } + + $redirectcount++; + + return $this->get($this->getabsoluteurl($headers["location"], $url), $reqtype, $acceptallcodes, $referer, $redirectcount); + }else{ + if( + $acceptallcodes === false && + $http["code"] > 300 + ){ + + throw new Exception("Remote server returned an error code! ({$http["code"]})"); + } + } + + // check if data is okay + switch($reqtype){ + + case self::req_image: + + $format = false; + + if(isset($headers["content-type"])){ + + if($headers["content-type"] == "text/html"){ + + throw new Exception("Server returned an html document instead of image"); + } + + $tmp = explode(";", $headers["content-type"]); + + for($i=0; $i $http, + "format" => $format, + "headers" => $headers, + "body" => $body + ]; + break; + + default: + + return [ + "http" => $http, + "headers" => $headers, + "body" => $body + ]; + break; + } + + return; + } + + public function stream_linear_image($url, $referer = null){ + + $this->stream($url, $referer, "image"); + } + + public function stream_linear_audio($url, $referer = null){ + + $this->stream($url, $referer, "audio"); + } + + private function stream($url, $referer, $format){ + + $this->url = $url; + $this->format = $format; + + // sanitize URL + try{ + + $this->validateurl($url); + }catch(Exception $error){ + + throw new Exception($error->getMessage()); + } + + $this->clientcache(); + + $curl = curl_init(); + + // set headers + if($referer === null){ + $referer = explode("/", $url, 4); + array_pop($referer); + + $referer = implode("/", $referer); + } + + switch($format){ + + case "image": + curl_setopt( + $curl, + CURLOPT_HTTPHEADER, + [ + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0", + "Accept: image/avif,image/webp,*/*", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate, br", + "DNT: 1", + "Connection: keep-alive", + "Referer: {$referer}" + ] + ); + break; + + case "audio": + curl_setopt( + $curl, + CURLOPT_HTTPHEADER, + [ + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0", + "Accept: audio/webm,audio/ogg,audio/wav,audio/*;q=0.9,application/ogg;q=0.7,video/*;q=0.6,*/*;q=0.5", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate, br", + "DNT: 1", + "Connection: keep-alive", + "Referer: {$referer}" + ] + ); + break; + } + + // follow redirects + curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($curl, CURLOPT_MAXREDIRS, 5); + curl_setopt($curl, CURLOPT_AUTOREFERER, 5); + + // set url + curl_setopt($curl, CURLOPT_URL, $url); + curl_setopt($curl, CURLOPT_ENCODING, ""); // default encoding + + // timeout + disable ssl + curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, 10); + curl_setopt($curl, CURLOPT_TIMEOUT, 30); + + curl_setopt( + $curl, + CURLOPT_WRITEFUNCTION, + function($c, $data){ + + if(curl_getinfo($c, CURLINFO_HTTP_CODE) !== 200){ + + throw new Exception("Serber returned a non-200 code"); + } + + echo $data; + return strlen($data); + } + ); + + $this->empty_header = false; + $this->cont = false; + $this->headers_tmp = []; + $this->headers = []; + curl_setopt( + $curl, + CURLOPT_HEADERFUNCTION, + function($c, $header){ + + $head = trim($header); + $len = strlen($head); + + if($len === 0){ + + $this->empty_header = true; + $this->headers_tmp = []; + }else{ + + $this->empty_header = false; + $this->headers_tmp[] = $head; + } + + foreach($this->headers_tmp as $h){ + + // parse headers + $h = explode(":", $h, 2); + + if(count($h) !== 2){ + + if(curl_getinfo($c, CURLINFO_HTTP_CODE) !== 200){ + + // not HTTP 200, probably a redirect + $this->cont = false; + }else{ + + $this->cont = true; + } + + // is HTTP 200, just ignore that line + continue; + } + + $this->headers[strtolower(trim($h[0]))] = trim($h[1]); + } + + if( + $this->cont && + $this->empty_header + ){ + + // get content type + if(isset($this->headers["content-type"])){ + + $filetype = explode("/", $this->headers["content-type"]); + + if(strtolower($filetype[0]) != $this->format){ + + throw new Exception("Resource is not an {$this->format} (Found {$filetype[0]} instead)"); + } + + }else{ + + throw new Exception("Resource is not an {$this->format} (no Content-Type)"); + } + + header("Content-Type: {$this->format}/{$filetype[1]}"); + + // give payload size + if(isset($this->headers["content-length"])){ + + header("Content-Length: {$this->headers["content-length"]}"); + } + + // give filename + $this->getfilenameheader($this->headers, $this->url, $filetype[1]); + } + + return strlen($header); + } + ); + + curl_exec($curl); + + if(curl_errno($curl)){ + + throw new Exception(curl_error($curl)); + } + + curl_close($curl); + } + + public function getfilenameheader($headers, $url, $filetype = "jpg"){ + + // get filename from content-disposition header + if(isset($headers["content-disposition"])){ + + preg_match( + '/filename=([^;]+)/', + $headers["content-disposition"], + $filename + ); + + if(isset($filename[1])){ + + header("Content-Disposition: filename=" . $filename[1] . "." . $filetype); + return; + } + } + + // get filename from URL + $filename = parse_url($url, PHP_URL_PATH); + + if($filename === null){ + + // everything failed! rename file to domain name + header("Content-Disposition: filename=" . parse_url($url, PHP_URL_HOST) . "." . $filetype); + return; + } + + // remove extension from filename + $filename = + explode( + ".", + basename($filename) + ); + + if(count($filename) > 1){ + array_pop($filename); + } + + $filename = implode(".", $filename); + + header("Content-Disposition: inline; filename=" . $filename . "." . $filetype); + return; + } + + public function getimageformat($payload, &$imagick){ + + $finfo = new finfo(FILEINFO_MIME_TYPE); + $format = $finfo->buffer($payload["body"]); + + if($format === false){ + + if($payload["format"] === false){ + + header("X-Error: Could not parse format"); + $this->favicon404(); + } + + $format = $payload["format"]; + }else{ + + $format_tmp = explode("/", $format, 2); + + if($format_tmp[0] == "image"){ + + $format_tmp = strtolower($format_tmp[1]); + + if(substr($format_tmp, 0, 2) == "x-"){ + + $format_tmp = substr($format_tmp, 2); + } + + $format = $format_tmp; + } + } + + switch($format){ + + case "tiff": $format = "gif"; break; + case "vnd.microsoft.icon": $format = "ico"; break; + case "icon": $format = "ico"; break; + case "svg+xml": $format = "svg"; break; + } + + $imagick = new Imagick(); + + if( + !in_array( + $format, + array_map("strtolower", $imagick->queryFormats()) + ) + ){ + + // format could not be found, but imagemagick can + // sometimes detect it? shit's fucked + $format = false; + } + + return $format; + } + + public function clientcache(){ + + if($this->cache === false){ + + return; + } + + header("Last-Modified: Thu, 01 Oct 1970 00:00:00 GMT"); + $headers = getallheaders(); + + if( + isset($headers["If-Modified-Since"]) || + isset($headers["If-Unmodified-Since"]) + ){ + + http_response_code(304); // 304: Not Modified + die(); + } + } +} diff --git a/lib/favicon404.png b/lib/favicon404.png new file mode 100644 index 0000000..7540694 Binary files /dev/null and b/lib/favicon404.png differ diff --git a/lib/frontend.php b/lib/frontend.php new file mode 100644 index 0000000..3be912b --- /dev/null +++ b/lib/frontend.php @@ -0,0 +1,1282 @@ + $value){ + + $html = + str_replace( + "{%{$key}%}", + $value, + $html + ); + } + + return trim($html); + } + + public function getthemeclass($raw = true){ + + if( + isset($_COOKIE["theme"]) && + $_COOKIE["theme"] == "cream" + ){ + + $body_class = "theme-white "; + }else{ + + $body_class = ""; + } + + if( + $raw && + $body_class != "" + ){ + + return ' class="' . rtrim($body_class) . '"'; + } + + return $body_class; + } + + public function loadheader(array $get, array $filters, string $page){ + + echo + $this->load("header.html", [ + "title" => trim($get["s"] . " ({$page})"), + "description" => ucfirst($page) . ' search results for "' . htmlspecialchars($get["s"]) . '"', + "index" => "no", + "search" => htmlspecialchars($get["s"]), + "tabs" => $this->generatehtmltabs($page, $get["s"]), + "filters" => $this->generatehtmlfilters($filters, $get), + "body_class" => $this->getthemeclass() + ]); + + if( + preg_match( + '/bot|wget|curl|python-requests|scrapy|feedfetcher|go-http-client|ruby|universalfeedparser|yahoo\! slurp|spider|rss/i', + $_SERVER["HTTP_USER_AGENT"] + ) + ){ + + // bot detected !! + echo + $this->drawerror( + "Tshh, blocked!", + 'You were blocked from viewing this page. If you wish to scrape data from 4get, please consider running your own 4get instance or using the API.', + ); + die(); + } + } + + public function drawerror($title, $error){ + + return + $this->load("search.html", [ + "class" => "", + "right-left" => "", + "right-right" => "", + "left" => + '

' . + '

' . htmlspecialchars($title) . '

' . + $error . + '

' + ]); + } + + public function drawtextresult($site, $greentext = null, $duration = null, $keywords, $tabindex = true){ + + $payload = + '

'; + + // add favicon, link and archive links + $payload .= $this->drawlink($site["url"]); + + /* + Draw title + description + filetype + */ + $payload .= + '' . + '

'; + + if($duration !== null){ + + $payload .= + '

' . + htmlspecialchars($duration) . + '

'; + } + + $payload .= + '

'; + + if( + isset($site["type"]) && + $site["type"] != "web" + ){ + + $payload .= '

' . strtoupper($site["type"]) . '

'; + } + + $payload .= + htmlspecialchars($site["title"]) . + '

'; + + if($greentext !== null){ + + $payload .= + '

' . + htmlspecialchars($greentext) . + '

'; + } + + if($site["description"] !== null){ + + $payload .= + '

' . + $this->highlighttext($keywords, $site["description"]) . + '

'; + } + + $payload .= ''; + + /* + Sublinks + */ + if( + isset($site["sublink"]) && + !empty($site["sublink"]) + ){ + + usort($site["sublink"], function($a, $b){ + + return strlen($a["description"]) > strlen($b["description"]); + }); + + $payload .= + '

' . + ''; + + $opentr = false; + for($i=0; $i' . + '' . + '

' . + htmlspecialchars($site["sublink"][$i]["title"]) . + '

'; + + if(!empty($site["sublink"][$i]["date"])){ + + $payload .= + '

' . + date("jS M y @ g:ia", $site["sublink"][$i]["date"]) . + '

'; + } + + if(!empty($site["sublink"][$i]["description"])){ + + $payload .= + '

' . + $this->highlighttext($keywords, $site["sublink"][$i]["description"]) . + '

'; + } + + $payload .= ''; + + if($opentr === false){ + + $payload .= ''; + } + } + + if($opentr === true){ + + $payload .= ''; + } + + $payload .= '

'; + } + + if( + isset($site["table"]) && + !empty($site["table"]) + ){ + + $payload .= ''; + + foreach($site["table"] as $title => $value){ + + $payload .= + '' . + '' . + '' . + ''; + } + + $payload .= '

' . htmlspecialchars($title) . '

' . htmlspecialchars($value) . '

'; + } + + return $payload . '

' . + '

' . htmlspecialchars($filter_values["display"]) . '

'; + + if(is_array($filter_values["option"])){ + + $tmp .= ''; + }else{ + + switch($filter_values["option"]){ + + case "_DATE": + $tmp .= ' $value){ + + if( + $value == null || + $value == false || + $key == "npt" || + $key == "extendedsearch" || + $value == "any" || + $value == "all" || + ( + $ommit === true && + $key == "s" + ) + ){ + + continue; + } + + $out[$key] = $value; + } + + return http_build_query($out); + } + + public function htmlnextpage($gets, $npt, $page){ + + $query = $this->buildquery($gets); + + return $page . "?" . $query . "&npt=" . $npt; + } +} diff --git a/lib/fuckhtml.php b/lib/fuckhtml.php new file mode 100644 index 0000000..8802511 --- /dev/null +++ b/lib/fuckhtml.php @@ -0,0 +1,361 @@ +load($html, $isfile); + } + } + + public function load($html, $isfile = false){ + + if(is_array($html)){ + + if(!isset($html["innerHTML"])){ + + throw new Exception("(load) Supplied array doesn't contain a innerHTML index"); + } + $html = $html["innerHTML"]; + } + + if($isfile){ + + $handle = fopen($html, "r"); + $fetch = fread($handle, filesize($html)); + fclose($handle); + + $this->html = $fetch; + }else{ + + $this->html = $html; + } + + $this->strlen = strlen($this->html); + } + + public function getElementsByTagName(string $tagname){ + + $out = []; + + /* + Scrape start of the tag. Example +

... + */ + + if($tagname == "*"){ + + $tagname = '[^\/<>\s]+'; + }else{ + + $tagname = preg_quote(strtolower($tagname)); + } + + preg_match_all( + '/<\s*(' . $tagname . ')(\s(?:[^>\'"]*|"[^"]*"|\'[^\']*\')+)?\s*>/i', + /* '/<\s*(' . $tagname . ')(\s[\S\s]*?)?>/i', */ + $this->html, + $starting_tags, + PREG_OFFSET_CAPTURE + ); + + for($i=0; $i strtolower($starting_tags[1][$i][0]), + "startPos" => $starting_tags[0][$i][1], + "endPos" => 0, + "startTag" => $starting_tags[0][$i][0], + "attributes" => $attributes, + "innerHTML" => null + ]; + } + + /* + Get innerHTML + */ + // get closing tag positions + preg_match_all( + '/<\s*\/\s*(' . $tagname . ')\s*>/i', + $this->html, + $regex_closing_tags, + PREG_OFFSET_CAPTURE + ); + + // merge opening and closing tags together + for($i=0; $i strtolower($regex_closing_tags[1][$i][0]), + "endTag" => $regex_closing_tags[0][$i][0], + "startPos" => $regex_closing_tags[0][$i][1] + ]; + } + + usort( + $out, + function($a, $b){ + + return $a["startPos"] > $b["startPos"]; + } + ); + + // computer the indent level for each element + $level = []; + $count = count($out); + + for($i=0; $i<$count; $i++){ + + if(!isset($level[$out[$i]["tagName"]])){ + + $level[$out[$i]["tagName"]] = 0; + } + + if(isset($out[$i]["startTag"])){ + + // encountered starting tag + $level[$out[$i]["tagName"]]++; + $out[$i]["level"] = $level[$out[$i]["tagName"]]; + }else{ + + // encountered closing tag + $out[$i]["level"] = $level[$out[$i]["tagName"]]; + $level[$out[$i]["tagName"]]--; + } + } + + // if the indent level is the same for a div, + // we encountered _THE_ closing tag + for($i=0; $i<$count; $i++){ + + if(!isset($out[$i]["startTag"])){ + + continue; + } + + for($k=$i; $k<$count; $k++){ + + if( + isset($out[$k]["endTag"]) && + $out[$i]["tagName"] == $out[$k]["tagName"] && + $out[$i]["level"] + === $out[$k]["level"] + ){ + + $startlen = strlen($out[$i]["startTag"]); + $endlen = strlen($out[$k]["endTag"]); + + $out[$i]["endPos"] = $out[$k]["startPos"] + $endlen; + + $out[$i]["innerHTML"] = + substr( + $this->html, + $out[$i]["startPos"] + $startlen, + $out[$k]["startPos"] - ($out[$i]["startPos"] + $startlen) + ); + + $out[$i]["outerHTML"] = + substr( + $this->html, + $out[$i]["startPos"], + $out[$k]["startPos"] - $out[$i]["startPos"] + $endlen + ); + + break; + } + } + } + + // filter out ending divs + for($i=0; $i<$count; $i++){ + + if(isset($out[$i]["endTag"])){ + + unset($out[$i]); + } + + unset($out[$i]["startTag"]); + } + + return array_values($out); + } + + public function getElementsByAttributeName(string $name, $collection = null){ + + if($collection === null){ + + $collection = $this->getElementsByTagName("*"); + }elseif(is_string($collection)){ + + $collection = $this->getElementsByTagName($collection); + } + + $return = []; + foreach($collection as $elem){ + + foreach($elem["attributes"] as $attrib_name => $attrib_value){ + + if($attrib_name == $name){ + + $return[] = $elem; + continue 2; + } + } + } + + return $return; + } + + public function getElementsByFuzzyAttributeValue(string $name, string $value, $collection = null){ + + $elems = $this->getElementsByAttributeName($name, $collection); + $value = explode(" ", $value); + + $return = []; + + foreach($elems as $elem){ + + foreach($elem["attributes"] as $attrib_name => $attrib_value){ + + $attrib_value = explode(" ", $attrib_value); + $ac = count($attrib_value); + $nc = count($value); + $cr = 0; + + for($i=0; $i<$nc; $i++){ + + for($k=0; $k<$ac; $k++){ + + if($value[$i] == $attrib_value[$k]){ + + $cr++; + } + } + } + + if($cr === $nc){ + + $return[] = $elem; + continue 2; + } + } + } + + return $return; + } + + public function getElementsByAttributeValue(string $name, string $value, $collection = null){ + + $elems = $this->getElementsByAttributeName($name, $collection); + + $return = []; + + foreach($elems as $elem){ + + foreach($elem["attributes"] as $attrib_name => $attrib_value){ + + if($attrib_value == $value){ + + $return[] = $elem; + continue 2; + } + } + } + + return $return; + } + + public function getElementById(string $idname, $collection = null){ + + $id = $this->getElementsByAttributeValue("id", $idname, $collection); + + if(count($id) !== 0){ + + return $id[0]; + } + + return false; + } + + public function getElementsByClassName(string $classname, $collection = null){ + + return $this->getElementsByFuzzyAttributeValue("class", $classname, $collection); + } + + public function getTextContent($html, $whitespace = false, $trim = true){ + + if(is_array($html)){ + + if(!isset($html["innerHTML"])){ + + throw new Exception("(getTextContent) Supplied array doesn't contain a innerHTML index"); + } + $html = $html["innerHTML"]; + } + + $html = + preg_split('/\n|<\/?br>/i', $html); + + $out = ""; + for($i=0; $i diff --git a/lib/img404.png b/lib/img404.png new file mode 100644 index 0000000..4549dee Binary files /dev/null and b/lib/img404.png differ diff --git a/lib/nextpage.php b/lib/nextpage.php new file mode 100644 index 0000000..a883e49 --- /dev/null +++ b/lib/nextpage.php @@ -0,0 +1,106 @@ +scraper = $scraper; + } + + public function store($payload, $page){ + + $page = $page[0]; + $password = random_bytes(256); // 2048 bit + $salt = random_bytes(16); + $key = hash_pbkdf2("sha512", $password, $salt, 20000, 32, true); + $iv = + random_bytes( + openssl_cipher_iv_length("aes-256-gcm") + ); + + $tag = ""; + $out = openssl_encrypt($payload, "aes-256-gcm", $key, OPENSSL_RAW_DATA, $iv, $tag, "", 16); + + $key = apcu_inc("key", 1); + + apcu_store( + $page . "." . + $this->scraper . + (string)($key), + gzdeflate($salt.$iv.$out.$tag), + 420 // cache information for 7 minutes blaze it + ); + + return + $this->scraper . $key . "." . + rtrim(strtr(base64_encode($password), '+/', '-_'), '='); + } + + public function get($npt, $page){ + + $page = $page[0]; + $explode = explode(".", $npt, 2); + + if(count($explode) !== 2){ + + throw new Exception("Malformed nextPageToken!"); + } + + $apcu = $page . "." . $explode[0]; + $key = $explode[1]; + + $payload = apcu_fetch($apcu); + + if($payload === false){ + + throw new Exception("The nextPageToken is invalid or has expired!"); + } + + $key = + base64_decode( + str_pad( + strtr($key, '-_', '+/'), + strlen($key) % 4, + '=', + STR_PAD_RIGHT + ) + ); + + $payload = gzinflate($payload); + + $key = + hash_pbkdf2( + "sha512", + $key, + substr($payload, 0, 16), // salt + 20000, + 32, + true + ); + $ivlen = openssl_cipher_iv_length("aes-256-gcm"); + + $payload = + openssl_decrypt( + substr( + $payload, + 16 + $ivlen, + -16 + ), + "aes-256-gcm", + $key, + OPENSSL_RAW_DATA, + substr($payload, 16, $ivlen), + substr($payload, -16) + ); + + if($payload === false){ + + throw new Exception("The nextPageToken is invalid or has expired!"); + } + + // remove the key after using + apcu_delete($apcu); + + return $payload; + } +} diff --git a/lib/type-todo.php b/lib/type-todo.php new file mode 100644 index 0000000..f813543 --- /dev/null +++ b/lib/type-todo.php @@ -0,0 +1,132 @@ + + public function type($get){ + + $search = $get["s"]; + $bang = $get["bang"]; + + if(empty($search)){ + + if(!empty($bang)){ + + // !youtube + $conn = pg_connect("host=localhost dbname=4get user=postgres password=postgres"); + + pg_prepare($conn, "bang_get", "SELECT bang,name FROM bangs WHERE bang LIKE $1 ORDER BY bang ASC LIMIT 8"); + $q = pg_execute($conn, "bang_get", ["$bang%"]); + + $results = []; + while($row = pg_fetch_array($q, null, PGSQL_ASSOC)){ + + $results[] = [ + "s" => "!" . $row["bang"], + "n" => $row["name"] + ]; + } + + return $results; + }else{ + + // everything is empty + // lets just return a bang list + return [ + [ + "s" => "!w", + "n" => "Wikipedia", + "u" => "https://en.wikipedia.org/wiki/Special:Search?search={%q%}" + ], + [ + "s" => "!4ch", + "n" => "4chan Board", + "u" => "https://find.4chan.org/?q={%q%}" + ], + [ + "s" => "!a", + "n" => "Amazon", + "u" => "https://www.amazon.com/s?k={%q%}" + ], + [ + "s" => "!e", + "n" => "eBay", + "u" => "https://www.ebay.com/sch/items/?_nkw={%q%}" + ], + [ + "s" => "!so", + "n" => "Stack Overflow", + "u" => "http://stackoverflow.com/search?q={%q%}" + ], + [ + "s" => "!gh", + "n" => "GitHub", + "u" => "https://github.com/search?utf8=%E2%9C%93&q={%q%}" + ], + [ + "s" => "!tw", + "n" => "Twitter", + "u" => "https://twitter.com/search?q={%q%}" + ], + [ + "s" => "!r", + "n" => "Reddit", + "u" => "https://www.reddit.com/search?q={%q%}" + ], + ]; + } + } + + // now we know search isnt empty + if(!empty($bang)){ + + // check if the bang exists + $conn = pg_connect("host=localhost dbname=4get user=postgres password=postgres"); + + pg_prepare($conn, "bang_get_single", "SELECT bang,name FROM bangs WHERE bang = $1 LIMIT 1"); + $q = pg_execute($conn, "bang_get_single", [$bang]); + + $row = pg_fetch_array($q, null, PGSQL_ASSOC); + + if(isset($row["bang"])){ + + $bang = "!$bang "; + }else{ + + $bang = ""; + } + } + + try{ + $res = $this->get( + "https://duckduckgo.com/ac/", + [ + "q" => strtolower($search) + ], + ddg::req_xhr + ); + + $res = json_decode($res, true); + + }catch(Exception $e){ + + throw new Exception("Failed to get /ac/"); + } + + $arr = []; + for($i=0; $i $res[$i]["phrase"] + ]; + }else{ + + $arr[] = [ + "s" => $bang . $res[$i]["phrase"], + "n" => $row["name"] + ]; + } + } + + return $arr; + } diff --git a/news.php b/news.php new file mode 100644 index 0000000..eb5817f --- /dev/null +++ b/news.php @@ -0,0 +1,96 @@ +getscraperfilters("news"); + +$get = $frontend->parsegetfilters($_GET, $filters); + +$frontend->loadheader( + $get, + $filters, + "news" +); + +$payload = [ + "class" => "", + "right-left" => "", + "right-right" => "", + "left" => "" +]; + +try{ + $results = $scraper->news($get); + +}catch(Exception $error){ + + echo + $frontend->drawerror( + "Shit", + 'This scraper returned an error:' . + '

' . htmlspecialchars($error->getMessage()) . '

' . + 'Things you can try:' . + '

Use a different scraper
Remove keywords that could cause errors
Use another 4get instance

' . + 'If the error persists, please contact the administrator.' + ); + die(); +} + +/* + Populate links +*/ +if(count($results["news"]) === 0){ + + $payload["left"] = + '

' . + "

Nobody here but us chickens!

" . + 'Have you tried:' . + '

Using a different scraper
Using fewer keywords
Defining broader filters (Is NSFW turned off?)

' . + '

'; +} + +foreach($results["news"] as $news){ + + $greentext = []; + + if($news["date"] !== null){ + + $greentext[] = date("jS M y @ g:ia", $news["date"]); + } + + if($news["author"] !== null){ + + $greentext[] = htmlspecialchars($news["author"]); + } + + if(count($greentext) !== 0){ + + $greentext = implode(" • ", $greentext); + }else{ + + $greentext = null; + } + + $n = null; + $payload["left"] .= $frontend->drawtextresult($news, $greentext, $n, $get["s"]); +} + +if($results["npt"] !== null){ + + $payload["left"] .= + 'Next page >'; +} + +echo $frontend->load("search.html", $payload); diff --git a/opensearch.xml b/opensearch.xml new file mode 100644 index 0000000..efce4b4 --- /dev/null +++ b/opensearch.xml @@ -0,0 +1,9 @@ + + +4get + +UTF-8 +https://4get.ca/favicon.ico + + + diff --git a/proxy.php b/proxy.php new file mode 100644 index 0000000..edefd77 --- /dev/null +++ b/proxy.php @@ -0,0 +1,130 @@ +do404(); + die(); +} + +try{ + + // original size request, stream file to browser + if( + !isset($_GET["s"]) || + $_GET["s"] == "original" + ){ + + $proxy->stream_linear_image($_GET["i"]); + die(); + } + + // bing request, ask bing to resize and stream to browser + if( + preg_match( + '/bing.net$/', + parse_url($_GET["i"], PHP_URL_HOST) + ) + ){ + + switch($_GET["s"]){ + + case "portrait": $req = "&w=50&h=90&p=0&qlt=99"; break; + case "landscape": $req = "&w=160&h=90&p=0&qlt=99"; break; + case "square": $req = "&w=90&h=90&p=0&qlt=99"; break; + case "thumb": $req = "&w=236&h=180&p=0&qlt=99"; break; + case "cover": $req = "&w=207&h=270&p=0&qlt=99"; break; + } + + $proxy->stream_linear_image($_GET["i"] . $req, "https://bing.net"); + die(); + } + + // resize image ourselves + $payload = $proxy->get($_GET["i"], $proxy::req_image, true); + + // get image format & set imagick + $image = null; + $format = $proxy->getimageformat($payload, $image); + + try{ + + if($format !== false){ + $image->setFormat($format); + } + + $image->readImageBlob($payload["body"]); + $image_width = $image->getImageWidth(); + $image_height = $image->getImageHeight(); + + switch($_GET["s"]){ + + case "portrait": + $width = 50; + $height = 90; + break; + + case "landscape": + $width = 160; + $height = 90; + break; + + case "square": + $width = 90; + $height = 90; + break; + + case "thumb": + $width = 236; + $height = 180; + break; + + case "cover": + $width = 207; + $height = 270; + break; + } + + $ratio = $image_width / $image_height; + + if($image_width > $width){ + + $image_width = $width; + $image_height = round($image_width / $ratio); + } + if($image_height > $height){ + + $ratio = $image_width / $image_height; + $image_height = $height; + $image_width = $image_height * $ratio; + } + + $image->setImageBackgroundColor(new ImagickPixel("#504945")); + $image->mergeImageLayers(Imagick::LAYERMETHOD_FLATTEN); + + $image->resizeImage($image_width, $image_height, Imagick::FILTER_LANCZOS, 1); + + $image->stripImage(); + $image->setFormat("jpeg"); + $image->setImageCompression(Imagick::COMPRESSION_JPEG2000); + + $proxy->getfilenameheader($payload["headers"], $_GET["i"]); + + header("Content-Type: image/jpeg"); + echo $image->getImageBlob(); + + }catch(ImagickException $error){ + + header("X-Error: Could not convert the image: (" . $error->getMessage() . ")"); + $proxy->do404(); + } + +}catch(Exception $error){ + + header("X-Error: " . $error->getMessage()); + $proxy->do404(); + die(); +} diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000..3e608cc --- /dev/null +++ b/robots.txt @@ -0,0 +1,28 @@ +# When the robots.txt is sus + +# ⠀⠀⠀⡯⡯⡾⠝⠘⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢊⠘⡮⣣⠪⠢⡑⡌ +# ⠀⠀⠀⠟⠝⠈⠀⠀⠀⠡⠀⠠⢈⠠⢐⢠⢂⢔⣐⢄⡂⢔⠀⡁⢉⠸⢨⢑⠕⡌ +# ⠀⠀⡀⠁⠀⠀⠀⡀⢂⠡⠈⡔⣕⢮⣳⢯⣿⣻⣟⣯⣯⢷⣫⣆⡂⠀⠀⢐⠑⡌ +# ⢀⠠⠐⠈⠀⢀⢂⠢⡂⠕⡁⣝⢮⣳⢽⡽⣾⣻⣿⣯⡯⣟⣞⢾⢜⢆⠀⡀⠀⠪ +# ⣬⠂⠀⠀⢀⢂⢪⠨⢂⠥⣺⡪⣗⢗⣽⢽⡯⣿⣽⣷⢿⡽⡾⡽⣝⢎⠀⠀⠀⢡ +# ⣿⠀⠀⠀⢂⠢⢂⢥⢱⡹⣪⢞⡵⣻⡪⡯⡯⣟⡾⣿⣻⡽⣯⡻⣪⠧⠑⠀⠁⢐ +# ⣿⠀⠀⠀⠢⢑⠠⠑⠕⡝⡎⡗⡝⡎⣞⢽⡹⣕⢯⢻⠹⡹⢚⠝⡷⡽⡨⠀⠀⢔ +# ⣿⡯⠀⢈⠈⢄⠂⠂⠐⠀⠌⠠⢑⠱⡱⡱⡑⢔⠁⠀⡀⠐⠐⠐⡡⡹⣪⠀⠀⢘ +# ⣿⣽⠀⡀⡊⠀⠐⠨⠈⡁⠂⢈⠠⡱⡽⣷⡑⠁⠠⠑⠀⢉⢇⣤⢘⣪⢽⠀⢌⢎ +# ⣿⢾⠀⢌⠌⠀⡁⠢⠂⠐⡀⠀⢀⢳⢽⣽⡺⣨⢄⣑⢉⢃⢭⡲⣕⡭⣹⠠⢐⢗ +# ⣿⡗⠀⠢⠡⡱⡸⣔⢵⢱⢸⠈⠀⡪⣳⣳⢹⢜⡵⣱⢱⡱⣳⡹⣵⣻⢔⢅⢬⡷ +# ⣷⡇⡂⠡⡑⢕⢕⠕⡑⠡⢂⢊⢐⢕⡝⡮⡧⡳⣝⢴⡐⣁⠃⡫⡒⣕⢏⡮⣷⡟ +# ⣷⣻⣅⠑⢌⠢⠁⢐⠠⠑⡐⠐⠌⡪⠮⡫⠪⡪⡪⣺⢸⠰⠡⠠⠐⢱⠨⡪⡪⡰ +# ⣯⢷⣟⣇⡂⡂⡌⡀⠀⠁⡂⠅⠂⠀⡑⡄⢇⠇⢝⡨⡠⡁⢐⠠⢀⢪⡐⡜⡪⡊ +# ⣿⢽⡾⢹⡄⠕⡅⢇⠂⠑⣴⡬⣬⣬⣆⢮⣦⣷⣵⣷⡗⢃⢮⠱⡸⢰⢱⢸⢨⢌ +# ⣯⢯⣟⠸⣳⡅⠜⠔⡌⡐⠈⠻⠟⣿⢿⣿⣿⠿⡻⣃⠢⣱⡳⡱⡩⢢⠣⡃⠢⠁ +# ⡯⣟⣞⡇⡿⣽⡪⡘⡰⠨⢐⢀⠢⢢⢄⢤⣰⠼⡾⢕⢕⡵⣝⠎⢌⢪⠪⡘⡌⠀ +# ⡯⣳⠯⠚⢊⠡⡂⢂⠨⠊⠔⡑⠬⡸⣘⢬⢪⣪⡺⡼⣕⢯⢞⢕⢝⠎⢻⢼⣀⠀ +# ⠁⡂⠔⡁⡢⠣⢀⠢⠀⠅⠱⡐⡱⡘⡔⡕⡕⣲⡹⣎⡮⡏⡑⢜⢼⡱⢩⣗⣯⣟ +# ⢀⢂⢑⠀⡂⡃⠅⠊⢄⢑⠠⠑⢕⢕⢝⢮⢺⢕⢟⢮⢊⢢⢱⢄⠃⣇⣞⢞⣞⢾ +# ⢀⠢⡑⡀⢂⢊⠠⠁⡂⡐⠀⠅⡈⠪⠪⠪⠣⠫⠑⡁⢔⠕⣜⣜⢦⡰⡎⡯⡾⡽ + +User-agent: * +Disallow: +host: 4get.ca +sitemap: https://4get.ca/sitemap.xml diff --git a/scraper/brave.php b/scraper/brave.php new file mode 100644 index 0000000..4d48c33 --- /dev/null +++ b/scraper/brave.php @@ -0,0 +1,2287 @@ +bypasscaptcha($html, "yes", "ca");*/ + +class brave{ + + public function __construct(){ + + include "lib/fuckhtml.php"; + $this->fuckhtml = new fuckhtml(); + + include "lib/nextpage.php"; + $this->nextpage = new nextpage("brave"); + } + + public function getfilters($page){ + + switch($page){ + + case "web": + return [ + "country" => [ + "display" => "Country", + "option" => [ + "all" => "All Regions", + "ar" => "Argentina", + "au" => "Australia", + "at" => "Austria", + "be" => "Belgium", + "br" => "Brazil", + "ca" => "Canada", + "cl" => "Chile", + "cn" => "China", + "dk" => "Denmark", + "fi" => "Finland", + "fr" => "France", + "de" => "Germany", + "hk" => "Hong Kong", + "in" => "India", + "id" => "Indonesia", + "it" => "Italy", + "jp" => "Japan", + "kr" => "Korea", + "my" => "Malaysia", + "mx" => "Mexico", + "nl" => "Netherlands", + "nz" => "New Zealand", + "no" => "Norway", + "pl" => "Poland", + "pt" => "Portugal", + "ph" => "Philippines", + "ru" => "Russia", + "sa" => "Saudi Arabia", + "za" => "South Africa", + "es" => "Spain", + "se" => "Sweden", + "ch" => "Switzerland", + "tw" => "Taiwan", + "tr" => "Turkey", + "gb" => "United Kingdom", + "us" => "United States" + ] + ], + "nsfw" => [ + "display" => "NSFW", + "option" => [ + "yes" => "Yes", + "maybe" => "Maybe", + "no" => "No" + ] + ], + "newer" => [ + "display" => "Newer than", + "option" => "_DATE" + ], + "older" => [ + "display" => "Older than", + "option" => "_DATE" + ] + ]; + break; + + case "news": + return [ + "country" => [ + "display" => "Country", + "option" => [ + "all" => "All regions", + "ar" => "Argentina", + "au" => "Australia", + "at" => "Austria", + "be" => "Belgium", + "br" => "Brazil", + "ca" => "Canada", + "cl" => "Chile", + "cn" => "China", + "dk" => "Denmark", + "fi" => "Finland", + "fr" => "France", + "de" => "Germany", + "hk" => "Hong Kong", + "in" => "India", + "id" => "Indonesia", + "it" => "Italy", + "jp" => "Japan", + "kr" => "Korea", + "my" => "Malaysia", + "mx" => "Mexico", + "nl" => "Netherlands", + "nz" => "New Zealand", + "no" => "Norway", + "pl" => "Poland", + "pt" => "Portugal", + "ph" => "Philippines", + "ru" => "Russia", + "sa" => "Saudi Arabia", + "za" => "South Africa", + "es" => "Spain", + "se" => "Sweden", + "ch" => "Switzerland", + "tw" => "Taiwan", + "tr" => "Turkey", + "gb" => "United Kingdom", + "us" => "United States" + ] + ], + "nsfw" => [ + "display" => "NSFW", + "option" => [ + "yes" => "Yes", + "maybe" => "Maybe", + "no" => "No" + ] + ] + ]; + break; + } + } + + private function get($url, $get = [], $nsfw, $country/*, $is_post = false, $additional_cookies = null*/){ + + switch($nsfw){ + + case "yes": $nsfw = "off"; break; + case "maybe": $nsfw = "moderate"; break; + case "no": $nsfw = "strict"; break; + } + + //$cookie = "safesearch={$nsfw}; country={$country}; useLocation=0"; + /* + if($additional_cookies !== null){ + + $cookie = $additional_cookies . "; " . $cookie; + }*/ + + $headers = [ + "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip", + "Cookie: safesearch={$nsfw}; country={$country}; useLocation=0; summarizer=0", + "DNT: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1"//, + //"Content-Type: application/json" + ]; + + if($country == "any"){ + + $country = "all"; + } + + $curlproc = curl_init(); + + /*if($is_post){ + + curl_setopt($curlproc, CURLOPT_POST, true); + curl_setopt( + $curlproc, + CURLOPT_POSTFIELDS, + json_encode($get) + ); + + }else{ + */ + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } + //} + + curl_setopt($curlproc, CURLOPT_URL, $url); + + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers); + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + return $data; + } + + public function web($get){ + + if($get["npt"]){ + + // get next page data + $q = json_decode($this->nextpage->get($get["npt"], "web"), true); + + $search = $q["q"]; + $q["spellcheck"] = 0; + + $nsfw = $q["nsfw"]; + unset($q["nsfw"]); + + $country = $q["country"]; + unset($q["country"]); + + }else{ + + // get _GET data instead + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + if(strlen($search) > 2048){ + + throw new Exception("Search query is too long!"); + } + + $nsfw = $get["nsfw"]; + $country = $get["country"]; + $older = $get["older"]; + $newer = $get["newer"]; + + $q = [ + "q" => $search + ]; + + /* + Pass older/newer filters to brave + */ + if($newer !== false){ + + $newer = date("Y-m-d", $newer); + + if($older === false){ + + $older = date("Y-m-d", time()); + } + } + + if( + is_string($older) === false && + $older !== false + ){ + + $older = date("Y-m-d", $older); + + if($newer === false){ + + $newer = "1970-01-02"; + } + } + + if($older !== false){ + + $q["tf"] = "{$newer}to{$older}"; + } + } + /* + $handle = fopen("scraper/brave.html", "r"); + $html = fread($handle, filesize("scraper/brave.html")); + fclose($handle); + */ + try{ + $html = + $this->get( + "https://search.brave.com/search", + $q, + $nsfw, + $country + ); + + }catch(Exception $error){ + + throw new Exception("Could not fetch search page"); + } + + $out = [ + "status" => "ok", + "spelling" => [ + "type" => "no_correction", + "using" => null, + "correction" => null + ], + "npt" => null, + "answer" => [], + "web" => [], + "image" => [], + "video" => [], + "news" => [], + "related" => [] + ]; + + // load html + $this->fuckhtml->load($html); + + /* + Get next page "token" + */ + $nextpage = + $this->fuckhtml + ->getElementsByClassName( + "btn ml-15", + "a" + ); + + if(count($nextpage) !== 0){ + + preg_match( + '/offset=([0-9]+)/', + $this->fuckhtml->getTextContent($nextpage[0]["attributes"]["href"]), + $nextpage + ); + + $q["offset"] = (int)$nextpage[1]; + $q["nsfw"] = $nsfw; + $q["country"] = $country; + + $out["npt"] = + $this->nextpage->store( + json_encode($q), + "web" + ); + } + + /* + Get discussions (and append them to web results) + */ + + // they're loaded using javascript!! + $discussion = + $this->fuckhtml + ->getElementById( + "js-discussions", + "script" + ); + + if( + $discussion && + isset($discussion["attributes"]["data"]) + ){ + + $discussion = + json_decode( + $this->fuckhtml + ->getTextContent( + $discussion["attributes"]["data"] + ), + true + ); + + foreach($discussion["results"] as $result){ + + $data = [ + "title" => $this->titledots($result["title"]), + "description" => null, + "url" => $result["url"], + "date" => null, + "type" => "web", + "thumb" => [ + "url" => null, + "ratio" => null + ], + "sublink" => [], + "table" => [] + ]; + + // description + $data["description"] = + $this->limitstrlen( + $this->limitwhitespace( + $this->titledots( + $this->fuckhtml->getTextContent( + $result["description"] + ) + ) + ) + ); + + if($result["age"] != ""){ + $data["date"] = strtotime($result["age"]); + } + + // populate table + + if($result["data"]["num_answers"] != ""){ + $data["table"]["Replies"] = (int)$result["data"]["num_answers"]; + } + + if($result["data"]["score"] != ""){ + + $score = explode("|", $result["data"]["score"]); + + if(count($score) === 2){ + + $score = ((int)$score[1]) . " (" . trim($score[0]) . ")"; + }else{ + + $score = (int)$score[0]; + } + + $data["table"]["Votes"] = $score; + } + + if($result["thumbnail"] != ""){ + + $data["thumb"]["url"] = $result["thumbnail"]; + $data["thumb"]["ratio"] = "16:9"; + } + + $out["web"][] = $data; + } + } + + /* + Get related searches + */ + $faq = + $this->fuckhtml + ->getElementById("js-faq", "script"); + + if( + $faq && + isset($faq["attributes"]["data"]) + ){ + + $faq = + json_decode( + $this->fuckhtml + ->getTextContent( + $faq["attributes"]["data"] + ), + true + ); + + foreach($faq["items"] as $related){ + + $out["related"][] = $related["question"]; + } + } + + /* + Get spelling autocorrect + */ + $altered = + $this->fuckhtml + ->getElementById("altered-query", "div"); + + if($altered){ + + $this->fuckhtml->load($altered); + + $altered = + $this->fuckhtml + ->getElementsByTagName("a"); + + if(count($altered) === 2){ + + $out["spelling"] = [ + "type" => "including", + "using" => + $this->fuckhtml + ->getTextContent($altered[0]), + "correction" => + $this->fuckhtml + ->getTextContent($altered[1]) + ]; + } + + $this->fuckhtml->load($html); + } + + /* + Get web results + */ + $resulthtml = + $this->fuckhtml + ->getElementById( + "results", + "div" + ); + + $this->fuckhtml->load($resulthtml); + $items = 0; + foreach( + $this->fuckhtml + ->getElementsByClassName("snippet fdb") + as $result + ){ + + $data = [ + "title" => null, + "description" => null, + "url" => null, + "date" => null, + "type" => "web", + "thumb" => [ + "url" => null, + "ratio" => null + ], + "sublink" => [], + "table" => [] + ]; + + if( + isset($result["attributes"]["data-type"]) && + $result["attributes"]["data-type"] == "ad" + ){ + + // is an ad, skip + continue; + } + + $this->fuckhtml->load($result); + + /* + Get title + */ + $title = + $this->fuckhtml + ->getElementsByClassName( + "snippet-title", + "span" + ); + + if(count($title) === 0){ + + // encountered AI summarizer + // or misspelling indicator @TODO + continue; + } + + if(isset($title[0]["attributes"]["title"])){ + + $data["title"] = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $title[0]["attributes"]["title"] + ) + ); + }else{ + + $data["title"] = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $title[0] + ) + ); + } + + /* + Get description + */ + $description = + $this->fuckhtml + ->getElementsByClassName( + "snippet-description", + "p" + ); + + if(count($description) !== 0){ + $data["description"] = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $description[0] + ) + ); + + // also check for thumbnail in here + $img = + $this->fuckhtml + ->getElementsByClassName( + "thumb", + "img" + ); + + if(count($img) !== 0){ + + $data["thumb"] = [ + "url" => $this->unshiturl($img[0]["attributes"]["src"]), + "ratio" => "16:9" + ]; + }else{ + + // might be a video thumbnail wrapper? + $wrapper = + $this->fuckhtml + ->getElementsByClassName( + "video-thumb", + "a" + ); + + if(count($wrapper) !== 0){ + + // we found a video + $this->fuckhtml->load($wrapper[0]); + + $img = + $this->fuckhtml + ->getElementsByTagName("img"); + + $data["thumb"] = [ + "url" => $this->unshiturl($img[0]["attributes"]["src"]), + "ratio" => "16:9" + ]; + + // get the video length, if its there + $duration = + $this->fuckhtml + ->getElementsByClassName( + "duration", + "div" + ); + + if(count($duration) !== 0){ + + $data["table"]["Duration"] = $duration[0]["innerHTML"]; + } + + // reset html load + $this->fuckhtml->load($result); + } + } + + }else{ + + // is a steam/shop listing + $description_alt = + $this->fuckhtml + ->getElementsByClassName( + "text-sm", + "div" + ); + + if(count($description_alt) !== 0){ + + switch($description_alt[0]["attributes"]["class"]){ + + case "text-sm text-gray": + case "description text-sm": + + $data["description"] = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $description_alt[0] + ) + ); + break; + } + + // get table sublink + $sublink = + $this->fuckhtml + ->getElementsByClassName( + "r-attr text-sm", + "div" + ); + + if(count($sublink) !== 0){ + + $this->tablesublink($sublink, $data); + } + + // check for thumb element + $data["thumb"] = $this->getimagelinkfromstyle("thumb"); + }else{ + + // ok... finally... + // maybe its the instant answer thingy + $answer = + $this->fuckhtml + ->getElementsByClassName("answer"); + + if(count($answer) !== 0){ + + $data["description"] = + $this->titledots( + $this->fuckhtml + ->getTextContent($answer[0]) + ); + } + } + } + + // finally, fix brave's date format sucking balls + $data["description"] = explode(" - ", $data["description"], 2); + + if(count($data["description"]) === 0){ + + // nothing to do + $data["description"] = $data["description"][0]; + }else{ + + // attempt to parse + $time = strtotime($data["description"][0]); + + if($time !== false){ + + // got response + $data["date"] = $time; + + array_shift($data["description"]); + } + + // merge back + $data["description"] = + implode(" - ", $data["description"]); + } + + /* + Check content type + */ + $content_type = + $this->fuckhtml + ->getElementsByClassName( + "content-type", + "span" + ); + + if(count($content_type) !== 0){ + + $data["type"] = + strtolower($this->fuckhtml->getTextContent($content_type[0])); + } + + /* + Check subtext table thingy + */ + $table_items = + array_merge( + $this->fuckhtml + ->getElementsByClassName( + "item-attributes", + "div" + ), + $this->fuckhtml + ->getElementsByClassName( + "r", + "div" + ) + ); + + /* + DIV: item-attributes + */ + if(count($table_items) !== 0){ + + foreach($table_items as $table){ + + $this->fuckhtml->load($table); + + $span = + $this->fuckhtml + ->getElementsByClassName( + "text-sm", + "*" + ); + + foreach($span as $item){ + + $item = + explode( + ":", + $this->fuckhtml->getTextContent(preg_replace('/\n/', " ", $item["innerHTML"])), + 2 + ); + + if(count($item) === 2){ + + $data["table"][trim($item[0])] = trim($this->limitwhitespace($item[1])); + } + } + } + + $this->fuckhtml->load($result); + } + + // get video sublinks + $table_items = + $this->fuckhtml + ->getElementsByClassName( + "snippet-description published-time", + "p" + ); + + if(count($table_items) !== 0){ + + $table_items = + explode( + '', + $table_items[0]["innerHTML"], + 2 + ); + if(count($table_items) === 2){ + + $item2 = []; + + $item2[] = explode(":", $this->fuckhtml->getTextContent($table_items[0])); + + if(trim($table_items[1]) != ""){ + $item2[] = explode(":", $this->fuckhtml->getTextContent($table_items[1])); + } + + foreach($item2 as $it){ + + $data["table"][trim($it[0])] = trim($it[1]); + } + } + } + + /* + Get URL + */ + $data["url"] = + $this->fuckhtml->getTextContent( + $this->fuckhtml + ->getElementsByTagName("a") + [0] + ["attributes"] + ["href"] + ); + + /* + Get sublinks + */ + $sublinks_elems = + $this->fuckhtml + ->getElementsByClassName( + "snippet", + "div" + ); + + $sublinks = []; + + foreach($sublinks_elems as $sublink){ + + $this->fuckhtml->load($sublink); + + $a = + $this->fuckhtml + ->getElementsByTagName("a")[0]; + + $title = + $this->fuckhtml + ->getTextContent($a); + + $url = $a["attributes"]["href"]; + + $description = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByTagName("p")[0] + ) + ); + + $sublinks[] = [ + "title" => $title, + "date" => null, + "description" => $description, + "url" => $url + ]; + } + + /* + Get smaller sublinks + */ + $sublinks_elems = + $this->fuckhtml + ->getElementsByClassName( + "deep-link", + "a" + ); + + foreach($sublinks_elems as $sublink){ + + $sublinks[] = [ + "title" => $this->fuckhtml->getTextContent($sublink), + "date" => null, + "description" => null, + "url" => $sublink["attributes"]["href"] + ]; + } + + // append sublinks to $data !! + $data["sublink"] = $sublinks; + + // append first result to start of $out["web"] + // other results are after + if($items === 0){ + + $out["web"] = [$data, ...$out["web"]]; + }else{ + + $out["web"][] = $data; + } + $items++; + } + + /* + Get news + */ + $this->fuckhtml->load($resulthtml); + $news_carousel = $this->fuckhtml->getElementById("news-carousel"); + + $this->fuckhtml->load($news_carousel); + + if($news_carousel){ + + $a = + $this->fuckhtml + ->getElementsByClassName( + "card fdb", + "a" + ); + + foreach($a as $news){ + + $this->fuckhtml->load($news); + + $out["news"][] = [ + "title" => + $this->titledots( + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "title", + "div" + )[0] + ) + ), + "description" => null, + "date" => + strtotime( + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "card-footer__timestamp", + "span" + )[0] + ) + ), + "thumb" => $this->getimagelinkfromstyle("img-bg"), + "url" => $this->fuckhtml->getTextContent($news["attributes"]["href"]) + ]; + } + } + + + + /* + Get videos + */ + $this->fuckhtml->load($resulthtml); + $news_carousel = $this->fuckhtml->getElementById("video-carousel"); + + $this->fuckhtml->load($news_carousel); + + if($news_carousel){ + + $a = + $this->fuckhtml + ->getElementsByClassName( + "card fdb", + "a" + ); + + foreach($a as $video){ + + $this->fuckhtml->load($video); + + $date = null; + + $date_o = + $this->fuckhtml + ->getElementsByClassName( + "text-gray text-xs", + "span" + ); + + if(count($date_o) !== 0){ + + $date = + strtotime( + $this->fuckhtml + ->getTextContent( + $date_o[0] + ) + ); + } + + $out["video"][] = [ + "title" => + $this->titledots( + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "title", + "div" + )[0] + ) + ), + "description" => null, + "date" => $date, + "duration" => null, + "views" => null, + "thumb" => $this->getimagelinkfromstyle("img-bg"), + "url" => $this->fuckhtml->getTextContent($video["attributes"]["href"]) + ]; + } + } + + + /* + Get DEFINITION snippet + */ + $this->fuckhtml->load($html); + $infobox = $this->fuckhtml->getElementById("rh-definitions", "div"); + + if($infobox !== false){ + + $answer = [ + "title" => null, + "description" => [], + "url" => null, + "thumb" => null, + "table" => [], + "sublink" => [] + ]; + + $this->fuckhtml->load($infobox); + + $answer["title"] = + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "header", + "h5" + )[0] + ); + + $sections = + $this->fuckhtml + ->getElementsByTagName("section"); + + $i = -1; + foreach($sections as $section){ + + $this->fuckhtml->load($section); + $items = + $this->fuckhtml + ->getElementsByTagName("*"); + + $li = 1; + $pronounce = false; + foreach($items as $item){ + + switch($item["tagName"]){ + + case "h6": + + if( + isset($item["attributes"]["class"]) && + $item["attributes"]["class"] == "h6 pronunciation" + ){ + + if($pronounce){ + + break; + } + + $answer["description"][] = [ + "type" => "quote", + "value" => + $this->fuckhtml + ->getTextContent( + $item + ) + ]; + + $answer["description"][] = + [ + "type" => "audio", + "url" => "https://search.brave.com/api/rhfetch?rhtype=definitions&word={$answer["title"]}&source=ahd-5" + ]; + + $pronounce = true; + $i = $i + 2; + break; + } + + $answer["description"][] = [ + "type" => "title", + "value" => + $this->fuckhtml + ->getTextContent( + $item + ) + ]; + $i++; + break; + + case "li": + + if( + $i !== -1 && + $answer["description"][$i]["type"] == "text" + ){ + + $answer["description"][$i]["value"] .= + "\n" . $li . ". " . + $this->fuckhtml + ->getTextContent( + $item + ); + + }else{ + $answer["description"][] = [ + "type" => "text", + "value" => + $li . ". " . + $this->fuckhtml + ->getTextContent( + $item + ) + ]; + $i++; + } + $li++; + break; + + case "a": + $answer["url"] = + $this->fuckhtml + ->getTextContent( + $item["attributes"]["href"] + ); + break; + } + } + } + + $out["answer"][] = $answer; + } + + + /* + Get instant answer + */ + $this->fuckhtml->load($html); + $infobox = $this->fuckhtml->getElementById("infobox", "div"); + + if($infobox !== false){ + + $answer = [ + "title" => null, + "description" => [], + "url" => null, + "thumb" => null, + "table" => [], + "sublink" => [] + ]; + + $this->fuckhtml->load($infobox); + $div = $this->fuckhtml->getElementsByTagName("div"); + + /* + Get title + url + */ + $title = + $this->fuckhtml + ->getElementsByClassName("infobox-title", "a"); + + if(count($title) !== 0){ + + $answer["title"] = + $this->fuckhtml + ->getTextContent( + $title[0] + ); + + $answer["url"] = + $this->fuckhtml + ->getTextContent( + $title[0]["attributes"]["href"] + ); + } + + /* + Get thumbnail + */ + $thumb = $this->getimagelinkfromstyle("thumb"); + + if($thumb["url"] !== null){ + + $answer["thumb"] = $thumb["url"]; + } + + /* + Get table + */ + $title = + $this->fuckhtml + ->getElementsByClassName( + "infobox-attr-header", + "div" + ); + + $rowhtml = $infobox; + + if(count($title) >= 2){ + + $rowhtml = + explode( + $title[1]["outerHTML"], + $infobox["innerHTML"], + 2 + )[0]; + } + + $this->fuckhtml->load($rowhtml); + + $rows = + $this->fuckhtml + ->getElementsByClassName("infobox-attr", "div"); + + foreach($rows as $row){ + + if(!isset($row["innerHTML"])){ + + continue; + } + + $this->fuckhtml->load($row); + $span = + $this->fuckhtml + ->getElementsByTagName("span"); + + if(count($span) === 2){ + + $answer["table"][ + $this->fuckhtml->getTextContent($span[0]) + ] = str_replace("\n", ", ", $this->fuckhtml->getTextContent($span[1], true)); + } + } + + $this->fuckhtml->load($infobox); + + /* + Parse stackoverflow answers + */ + $code = + $this->fuckhtml + ->getElementById("codebox-answer", $div); + + if($code){ + + // this might be standalone text with no paragraphs, check for that + $author = + $this->fuckhtml + ->getElementById("author"); + + $desc_tmp = + str_replace( + $author["outerHTML"], + "", + $code["innerHTML"] + ); + + $this->fuckhtml->load($desc_tmp); + $code = + $this->fuckhtml + ->getElementsByTagName("*"); + + if(count($code) === 0){ + + $answer["description"] = + [ + [ + "type" => "text", + "value" => + $this->fuckhtml + ->getTextContent( + $desc_tmp + ) + ], + [ + "type" => "quote", + "value" => + $this->fuckhtml + ->getTextContent( + $author + ) + ] + ]; + }else{ + + $text = []; + $i = 0; + + foreach($code as $snippet){ + + switch($snippet["tagName"]){ + + case "p": + $this->fuckhtml->load($snippet["innerHTML"]); + + $codetags = + $this->fuckhtml + ->getElementsByTagName("*"); + + $tmphtml = $snippet["innerHTML"]; + + foreach($codetags as $tag){ + + if(!isset($tag["outerHTML"])){ + + continue; + } + + $tmphtml = + explode( + $tag["outerHTML"], + $tmphtml, + 2 + ); + + $value = $this->fuckhtml->getTextContent($tmphtml[0], false, false); + $this->appendtext($value, $text, $i); + + $type = null; + switch($tag["tagName"]){ + + case "code": $type = "inline_code"; break; + case "em": $type = "italic"; break; + case "blockquote": $type = "quote"; break; + default: $type = "text"; + } + + if($type !== null){ + $value = $this->fuckhtml->getTextContent($tag, false, true); + + if(trim($value) != ""){ + + if( + $i !== 0 && + $type == "title" + ){ + + $text[$i - 1]["value"] = rtrim($text[$i - 1]["value"]); + } + + $text[] = [ + "type" => $type, + "value" => $value + ]; + $i++; + } + } + + if(count($tmphtml) === 2){ + + $tmphtml = $tmphtml[1]; + }else{ + + break; + } + } + + if(is_array($tmphtml)){ + + $tmphtml = $tmphtml[0]; + } + + if(strlen($tmphtml) !== 0){ + + $value = $this->fuckhtml->getTextContent($tmphtml, false, false); + $this->appendtext($value, $text, $i); + } + break; + + case "pre": + + switch($text[$i - 1]["type"]){ + + case "text": + case "italic": + $text[$i - 1]["value"] = rtrim($text[$i - 1]["value"]); + break; + } + + $text[] = + [ + "type" => "code", + "value" => + rtrim( + $this->fuckhtml + ->getTextContent( + $snippet, + true, + false + ) + ) + ]; + $i++; + + break; + + case "ol": + $o = 0; + + $this->fuckhtml->load($snippet); + $li = + $this->fuckhtml + ->getElementsByTagName("li"); + + foreach($li as $elem){ + $o++; + + $this->appendtext( + $o . ". " . + $this->fuckhtml + ->getTextContent( + $elem + ), + $text, + $i + ); + } + break; + } + } + + if( + $i !== 0 && + $text[$i - 1]["type"] == "text" + ){ + + $text[$i - 1]["value"] = rtrim($text[$i - 1]["value"]); + } + + if($author){ + + $text[] = [ + "type" => "quote", + "value" => $this->fuckhtml->getTextContent($author) + ]; + } + + $answer["description"] = $text; + } + }else{ + + /* + Get normal description + */ + $description = + $this->fuckhtml + ->getElementsByClassName( + "mb-6", + "div" + ); + + if(count($description) !== 0){ + + $description = + [ + [ + "type" => "text", + "value" => + $this->titledots( + preg_replace( + '/ Wikipedia$/', + "", + $this->fuckhtml + ->getTextContent( + $description[0] + ) + ) + ) + ] + ]; + + $ratings = + $this->fuckhtml + ->getElementById("ratings"); + + if($ratings){ + + $this->fuckhtml->load($ratings); + + $ratings = + $this->fuckhtml + ->getElementsByClassName( + "flex-hcenter mb-10", + "div" + ); + + $description[] = [ + "type" => "title", + "value" => "Ratings" + ]; + + foreach($ratings as $rating){ + + $this->fuckhtml->load($rating); + + $num = + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "r-num", + "div" + )[0] + ); + + $href = + $this->fuckhtml + ->getElementsByClassName( + "mr-10", + "a" + )[0]; + + $votes = + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "text-sm", + "span" + )[0] + ); + + $c = count($description) - 1; + + if( + $c !== -1 && + $description[$c]["type"] == "text" + ){ + + $description[$c]["value"] .= $num . " "; + }else{ + + $description[] = [ + "type" => "text", + "value" => $num . " " + ]; + } + + $description[] = [ + "type" => "link", + "value" => $this->fuckhtml->getTextContent($href), + "url" => $this->fuckhtml->getTextContent($href["attributes"]["href"]) + ]; + + $description[] = [ + "type" => "text", + "value" => " (" . $votes . ")\n" + ]; + } + } + + $answer["description"] = $description; + } + } + + /* + Get sublinks + */ + $this->fuckhtml->load($infobox); + + $profiles = + $this->fuckhtml + ->getElementById("profiles"); + + if($profiles){ + $profiles = + $this->fuckhtml + ->getElementsByClassName( + "chip", + "a" + ); + + foreach($profiles as $profile){ + + $name = $this->fuckhtml->getTextContent($profile["attributes"]["title"]); + + if(strtolower($name) == "steampowered"){ + + $name = "Steam"; + } + + $answer["sublink"][$name] = + $this->fuckhtml->getTextContent($profile["attributes"]["href"]); + } + } + + $actors = + $this->fuckhtml + ->getElementById("panel-movie-cast"); + + if($actors){ + + $this->fuckhtml->load($actors); + + $actors = + $this->fuckhtml + ->getElementsByClassName("card"); + + $answer["description"][] = [ + "type" => "title", + "value" => "Cast" + ]; + + foreach($actors as $actor){ + + $this->fuckhtml->load($actor); + + $answer["description"][] = [ + "type" => "text", + "value" => + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName("card-body") + [0] + ) + ]; + + $answer["description"][] = [ + "type" => "image", + "url" => $this->getimagelinkfromstyle("person-thumb")["url"] + ]; + } + } + + $out["answer"][] = $answer; + } + + /* + Get actor standalone thingy + */ + $this->fuckhtml->load($resulthtml); + $actors = + $this->fuckhtml + ->getElementById("predicate-entity"); + + if($actors){ + + $this->fuckhtml->load($actors); + + $cards = + $this->fuckhtml + ->getElementsByClassName("card"); + + $url = + $this->fuckhtml + ->getElementsByClassName( + "disclaimer", + "div" + )[0]; + + $this->fuckhtml->load($url); + + $url = + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByTagName("a") + [0] + ["attributes"] + ["href"] + ); + + $this->fuckhtml->load($actors); + + $answer = [ + "title" => + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "entity", + "span" + )[0] + ) . " (Cast)", + "description" => [], + "url" => $url, + "sublink" => [], + "thumb" => null, + "table" => [] + ]; + + foreach($cards as $card){ + + $this->fuckhtml->load($card); + + $answer["description"][] = [ + "type" => "title", + "value" => + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "title" + )[0] + ) + ]; + + $answer["description"][] = [ + "type" => "text", + "value" => + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "text-xs desc" + )[0] + ) + ]; + + $answer["description"][] = [ + "type" => "image", + "url" => $this->getimagelinkfromstyle("img-bg")["url"] + ]; + } + + $out["answer"][] = $answer; + } + + return $out; + } + + public function news($get){ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $nsfw = $get["nsfw"]; + $country = $get["country"]; + + if(strlen($search) > 2048){ + + throw new Exception("Search query is too long!"); + } + /* + $handle = fopen("scraper/brave-news.html", "r"); + $html = fread($handle, filesize("scraper/brave-news.html")); + fclose($handle);*/ + try{ + $html = + $this->get( + "https://search.brave.com/news", + [ + "q" => $search + ], + $nsfw, + $country + ); + + }catch(Exception $error){ + + throw new Exception("Could not fetch search page"); + } + + $out = [ + "status" => "ok", + "npt" => null, + "news" => [] + ]; + + // load html + $this->fuckhtml->load($html); + + $news = + $this->fuckhtml + ->getElementsByClassName( + "snippet inline gap-standard", + "div" + ); + + foreach($news as $article){ + + $data = [ + "title" => null, + "author" => null, + "description" => null, + "date" => null, + "thumb" => + [ + "url" => null, + "ratio" => null + ], + "url" => null + ]; + + $this->fuckhtml->load($article); + $elems = + $this->fuckhtml + ->getElementsByTagName("*"); + + // get title + $data["title"] = + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "snippet-title", + $elems + ) + [0] + ["innerHTML"] + ); + + // get description + $data["description"] = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "snippet-description", + $elems + ) + [0] + ["innerHTML"] + ) + ); + + // get date + $date = + explode( + "•", + $this->fuckhtml + ->getTextContent( + $this->fuckhtml + ->getElementsByClassName( + "snippet-url", + $elems + )[0] + ) + ); + + if( + count($date) !== 1 && + trim($date[1]) != "" + ){ + + $data["date"] = + strtotime( + $date[1] + ); + } + + // get URL + $data["url"] = + $this->fuckhtml->getTextContent( + $this->unshiturl( + $this->fuckhtml + ->getElementsByClassName( + "result-header", + $elems + ) + [0] + ["attributes"] + ["href"] + ) + ); + + // get thumbnail + $thumb = + $this->fuckhtml + ->getElementsByTagName( + "img" + ); + + if( + count($thumb) === 2 && + trim( + $thumb[1] + ["attributes"] + ["src"] + ) != "" + ){ + + $data["thumb"] = [ + "url" => + $this->fuckhtml->getTextContent( + $this->unshiturl( + $thumb[1] + ["attributes"] + ["src"] + ) + ), + "ratio" => "16:9" + ]; + } + + $out["news"][] = $data; + } + + return $out; + } + + /* + public function bypasscaptcha($html, $nsfw, $country){ + + // @TODO figure out why I still cant go trough + // the captcha wall even after breaking it + + try{ + $html = + $this->get( + "https://search.brave.com/goggles", + [ + "q" => "site:dailymotion.com my bloody valentine" + ], + $nsfw, + $country + ); + + }catch(Exception $error){ + + throw new Exception("Could not fetch html"); + } + + // Bypass brave search captcha + // this captcha only appears on the goggles page + preg_match( + '/this\.img\.src = "(.*)"/', + $html, + $image + ); + + $image = + base64_decode( + explode( + "data:image/png;base64,", + $image[1] + )[1] + ); + + $im = new Imagick(); + $im->readImageBlob($image); + + $im->blurImage(20, 20); + $im->posterizeImage(2, imagick::IMGTYPE_COLORSEPARATION); + + // if we encounter a white line thats longer than 45px + // we found the circle position + $iterator = $im->getPixelRegionIterator(0, 77, 310, 1); + + $found = null; + foreach( + $iterator as $row + ){ + + $whitecount = 0; + $count = 0; + + foreach($row as $pixel){ + + if($pixel->getColor()["r"] === 255){ + + $whitecount++; + $pixel->setColor("rgba(255,0,0,0)"); + + if($whitecount === 45){ + + $found = $count - 45; + break 2; + } + }else{ + + $whitecount = 0; + } + + $count++; + $iterator->syncIterator(); + } + } + + $found = $found + 10; + + //header("Content-Type: image/png"); + //echo $im; + //die(); + + if($found === null){ + + throw new Exception("Could not bypass captcha"); + } + + preg_match( + '/data="{"captcha_id":"([0-9A-z-]+)"}"/', + $html, + $key + ); + + $key = $key[1]; + // we bypassed captcha, send POST data + $order = + $this->get( + "https://search.brave.com/api/captcha?brave=0&captcha_id={$key}", + [ + "solution" => (string)$found + ], + $nsfw, + $country, + true + ); + + $order = json_decode($order, true)["orderId"]; + + $orderpayload = + $this->get( + "https://search.brave.com/api/rewards/v1/orders/{$order}", + [], + $nsfw, + $country + ); + + $orderpayload = json_decode($orderpayload, true); + + $creds = + $this->get( + "https://search.brave.com/api/rewards/v1/orders/{$order}/credentials", + [ + "itemId" => $orderpayload["items"][0]["id"], + "blindedCreds" => [ + "fuYAVcB/m7BU66vf3wkNGxJCSaRhshB9o+8km3F1h2c=", + "uswvcWJuPK/1qFlVdzBP3eQd0+V1EQgfAtnEoMIK+Uk=", + "fJWKGLBxl3Gyn4n9FjTLq1PjupfABT7Ni8MeB+iGzUs=", + "Aq9enJ/VZP9GxQIza3n65ZK7xQhY4VwDxv53BCb/Txg=", + "FMJA9eSLHq71K+Pcwgm4gIQOmdR/6KMy5cMgXhpd5Ro=", + "2NVhIAbvI317SP9/xXbVe/U57eWgvHyqVbHL/5+Gdmw=", + "6mpjsjSCmYEzK2xlbL8DI2P4LuhWUOxjTLvsTAL9l24=", + "kAn4wuHvIlKWhfuFfPTSfD4tZ5le9t7/61YbdEc/L3k=", + "BjjUyG16aTfd1c0h4oBzgQQOekrH1f+a5CmcXqMPTR4=", + "SBNgpCt4/V44yaQTfh+D027Yv1GJFHkjUEpPw6rAwRI=", + "XDENAtdQ7PyYx+Qx1wQGQtDWgg8WpIMgWGmd4RDOVWE=", + "tF7rB4sqamsiUk3K7fojdQSI0Q6iip72yKyhnvg/bC0=", + "VsAqflirAd/u4VsLdfRS2UvnH24ZNkFh6YN3DctLjzQ=", + "MntLbXkoI0LdcisCbNazmooiHXJyX91L1KERDAu1JRU=", + "TH6Zs8JBvFDbTDWgKbfGE4M5/cSwCtHD8ms5Y/U8zHQ=", + "jsZg0Z+qDPHymrbhdnesodhLNJ26QdunyMko1aVe4So=", + "rpKsyj6/vdnuMgLI2BApeijtGq9g5USRDL0w6X2bnlQ=", + "vCzliGT8A9vcLXj2sFf2kavOuYw69d70NpfgA22B4lI=", + "7OWoxSCtYXWcaBSifF7AXNBif/sjcuO0IelzXG/3PFk=", + "iiXtByNlT6nDMN9De5B58Jl8J0p6LCjnZ9aS3w2FEQU=", + "zDhd7gsJ4h4JkDeGK0Y0mfFd8IBdkLhMOANzwO+4Dig=", + "qANZ+AikwFReEA61JF009d/c3IHM/aSfIYwljckhJWE=", + "nNC30pDLxtXvUr+WDwfDSrAInNBpfSZkPsV2JlpheWI=", + "kGXE1pkt25P71kdJzmKIg4+yMR1VA5wNmbpBb/FhJQ8=", + "aLqPsY1Qiz2UCa2Jx3YNNt8r4JINMphks/43EiyZfXU=", + "bHGYZoQARZEM5LdFF6B74PkRqNd9EKxzuTvGYxjq+hk=", + "JOsYQjfE/9Y1u29hR+GvEkNyxUI8blgLhX1iJI/aGRQ=", + "yKjHjH5j600TJD/3WPsA1N3OmItDLifdjlysq4H6NV0=", + "9lTnUbsPp7BJ7XVN5/T4yGfzD9DJdqWB7xk72s19MAA=", + "5KHG8iY45em7zDhO/HlI0ydcZ0Ubn+XSyjifMmy7qXM=" + ] + ], + $nsfw, + $country, + true + ); + + var_dump($creds); + + sleep(2); + $test = + $this->get( + "https://search.brave.com/api/rewards/v1/orders/{$order}/credentials", + [], + $nsfw, + $country + ); + + var_dump($test); + + $html = + $this->get( + "https://search.brave.com/goggles", + [ + "q" => "site:dailymotion.com my bloody valentine" + ], + $nsfw, + $country, + false, + "__Secure-sku#brave-search-captcha=eyJ0eXBlIjoic2luZ2xlLXVzZSIsInZlcnNpb24iOjEsInNrdSI6ImJyYXZlLXNlYXJjaC1jYXB0Y2hhIiwicHJlc2VudGF0aW9uIjoiZXlKcGMzTjFaWElpT2lKaWNtRjJaUzVqYjIwL2MydDFQV0p5WVhabExYTmxZWEpqYUMxallYQjBZMmhoSWl3aWMybG5ibUYwZFhKbElqb2lNRzl0VDBneWQxZ3dTazkzU0VFMVJ6QTJaR1V5WjFOQ1dDdGhSM3B2Y2xsTVQwVTJZVVJtTUc5a1IweG1Wa3RhZEd0cU4xbHdia3BPT0VOVGNGbE5lVWR2YmpGRlNTOUhhMlZYU1RWNGQxTjJPWGxJTTNjOVBTSXNJblFpT2lKWlJWWldaVzR5TTJwQ01tSnZkakJ2U1hGNGJtSndUMGxEUW5Kd1drRjBRbWQxVnpoRlNURTNVREY2UVRaQlpUTXJSVGRFYm5NeVFqUmhka0pGYTFWM2FGY3JWRVZJVjNWcE9TdFllRU1yYlVSTVkyMTBRVDA5SW4wPSJ9" + ); + + var_dump($html); + }*/ + + private function appendtext($payload, &$text, &$index){ + + if(trim($payload) == ""){ + + return; + } + + if( + $index !== 0 && + $text[$index - 1]["type"] == "text" + ){ + + $text[$index - 1]["value"] .= "\n\n" . preg_replace('/ $/', " ", $payload); + }else{ + + $text[] = [ + "type" => "text", + "value" => preg_replace('/ $/', " ", $payload) + ]; + $index++; + } + } + + private function tablesublink($html_collection, &$data){ + + foreach($html_collection as $html){ + + $html["innerHTML"] = preg_replace( + '/

Field	Value
Search engine	4get
Shortcut	4get.ca
URL with %s in place of query	https://4get.ca/web?q=%s

Set as default search engine

On Firefox and other Gecko based browsers

On Chromium and Blink based browsers

Other browsers

Frequently asked questions

What is this?

My goal

Do you keep logs?

Where is this website hosted?

Keyboard shortcuts?

Instances

How can I trust you?

I want to report abuse or have erotic roleplay trough email

Nobody here but us chickens!

' . htmlspecialchars($title) . '

Nobody here but us chickens!

Name	Address
4get	4get.ca (tor)

Set as default search engine

On Firefox and other Gecko based browsers

On Chromium and Blink based browsers

Other browsers

Frequently asked questions

What is this?

My goal

Do you keep logs?

Do you share information with third parties?

Where is this website hosted?

Keyboard shortcuts?

Instances

How can I trust you?

I want to report abuse or have erotic roleplay trough email

Nobody here but us chickens!

' . htmlspecialchars($title) . '

Nobody here but us chickens!