1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579 |
- <?php
- class startpage{
-
- public function __construct(){
-
- include "lib/backend.php";
- $this->backend = new backend("startpage");
-
- include "lib/fuckhtml.php";
- $this->fuckhtml = new fuckhtml();
- }
-
- public function getfilters($page){
-
- switch($page){
- case "web":
- return [
- "country" => [
- "display" => "Country",
- "option" => [
- "any" => "All Regions",
- "es_AR" => "Argentina",
- "en_AU" => "Australia",
- "de_AT" => "Austria",
- "ru_BY" => "Belarus",
- "fr_BE" => "Belgium (FR)",
- "nl_BE" => "Belgium (NL)",
- "bg_BG" => "Bulgaria",
- "en_CA" => "Canada (EN)",
- "fr_CA" => "Canada (FR)",
- "es_CL" => "Chile",
- "es_CO" => "Colombia",
- "cs_CZ" => "Czech Republic",
- "da_DK" => "Denmark",
- "ar_EG" => "Egypt",
- "et_EE" => "Estonia",
- "fi_FI" => "Finland",
- "fr_FR" => "France",
- "de_DE" => "Germany",
- "el_GR" => "Greece",
- "hu_HU" => "Hungary",
- "hi_IN" => "India (HI)",
- "en_IN" => "India (EN)",
- "id_ID" => "Indonesia (ID)",
- "en_ID" => "Indonesia (EN)",
- "en_IE" => "Ireland",
- "it_IT" => "Italy",
- "ja_JP" => "Japan",
- "ko_KR" => "Korea",
- "ms_MY" => "Malaysia (MS)",
- "en_MY" => "Malaysia (EN)",
- "es_MX" => "Mexico",
- "nl_NL" => "Netherlands",
- "en_NZ" => "New Zealand",
- "no_NO" => "Norway",
- "es_PE" => "Peru",
- "fil_PH" => "Philippines (FIL)",
- "en_PH" => "Philippines (EN)",
- "pl_PL" => "Poland",
- "pt_PT" => "Portugal",
- "ro_RO" => "Romania",
- "ru_RU" => "Russia",
- "ms_SG" => "Singapore (MS)",
- "en_SG" => "Singapore (EN)",
- "es_ES" => "Spain (ES)",
- "ca_ES" => "Spain (CA)",
- "sv_SE" => "Sweden",
- "de_CH" => "Switzerland (DE)",
- "fr_CH" => "Switzerland (FR)",
- "it_CH" => "Switzerland (IT)",
- "tr_TR" => "Turkey",
- "uk_UA" => "Ukraine",
- "en_US" => "US (EN)",
- "es_US" => "US (ES)",
- "es_UY" => "Uruguay",
- "es_VE" => "Venezuela",
- "vi_VN" => "Vietnam (VI)",
- "en_VN" => "Vietnam (EN)",
- "en_ZA" => "South Africa"
- ]
- ],
- "nsfw" => [
- "display" => "NSFW",
- "option" => [
- "yes" => "Yes",
- "no" => "No"
- ]
- ],
- "time" => [
- "display" => "Time posted",
- "option" => [
- "any" => "Any time",
- "d" => "Past 24 hours",
- "w" => "Past week",
- "m" => "Past month",
- "y" => "Past year",
- ]
- ],
- "extendedsearch" => [
-
- "option" => [
- "yes" => "Yes",
- "no" => "No"
- ]
- ]
- ];
- break;
-
- case "images":
- return [
- "nsfw" => [
- "display" => "NSFW",
- "option" => [
- "yes" => "Yes",
- "no" => "No"
- ]
- ],
- "size" => [
- "display" => "Size",
- "option" => [
- "any" => "Any size",
- "Small" => "Small",
- "Medium" => "Medium",
- "Large" => "Large",
- "Wallpaper" => "Wallpaper",
-
- "qsvgs" => "Larger than 400x300",
- "vga" => "Larger than 640x480",
- "svga" => "Larger than 800x600",
- "xga" => "Larger than 1024x768",
- "qsvgs" => "Larger than 400x300",
- "2mp" => "Larger than 2 MP (1600x1200)",
- "4mp" => "Larger than 4 MP (2272x1704)",
- "6mp" => "Larger than 6 MP (2816x2112)",
- "8mp" => "Larger than 8 MP (3264x2448)",
- "10mp" => "Larger than 10 MP (3648x2736)",
- "12mp" => "Larger than 12 MP (4096x3072)",
- "15mp" => "Larger than 15 MP (4480x3360)",
- "20mp" => "Larger than 20 MP (5120x3840)",
- "40mp" => "Larger than 40 MP (7216x5412)",
- "70mp" => "Larger than 70 MP (9600x7200)"
- ]
- ],
- "color" => [
- "display" => "Color",
- "option" => [
- "any" => "Any color",
-
- "color" => "Color only",
- "bnw" => "Black & white",
-
- "red" => "Red",
- "orange" => "Orange",
- "yellow" => "Yellow",
- "green" => "Green",
- "teal" => "Teal",
- "blue" => "Blue",
- "purple" => "Purple",
- "pink" => "Pink",
- "white" => "White",
- "gray" => "Gray",
- "black" => "Black",
- "brown" => "Brown"
- ]
- ],
- "type" => [
- "display" => "Type",
- "option" => [
- "any" => "Any type",
- "AnimatedGif" => "Animated GIF",
- "Clipart" => "Clip Art",
- "Line" => "Line Drawing",
- "Photo" => "Photograph",
- "Transparent" => "Transparent Background"
- ]
- ],
- "license" => [
- "display" => "License",
- "option" => [
- "any" => "Any license",
- "p" => "Public domain",
- "s" => "Free to share",
- "sc" => "Free to share commercially",
- "m" => "Free to modify",
- "mc" => "Free to modify commercially"
- ]
- ]
- ];
- break;
-
- case "videos":
- return [
- "nsfw" => [
- "display" => "NSFW",
- "option" => [
- "yes" => "Yes",
- "no" => "No"
- ]
- ],
- "sort" => [
- "display" => "Sort by",
- "option" => [
- "relevance" => "Most relevant",
- "popular" => "Most popular",
- "recent" => "Most recent"
- ]
- ],
- "duration" => [
- "display" => "Duration",
- "option" => [
- "any" => "Any duration",
- "short" => "Short",
- "medium" => "Medium",
- "long" => "Long"
- ]
- ]
- ];
- break;
-
- case "news":
- return [
- "nsfw" => [
- "display" => "NSFW",
- "option" => [
- "yes" => "Yes",
- "no" => "No"
- ]
- ],
- "time" => [
- "display" => "Time posted",
- "option" => [
- "any" => "Any time",
- "d" => "Past 24 hours",
- "w" => "Past week",
- "m" => "Past month"
- ]
- ]
- ];
- break;
-
-
-
- }
- }
-
- private function get($proxy, $url, $get = [], $post = false, $is_xhr = false){
-
- $curlproc = curl_init();
-
- if($post === true){
-
- curl_setopt($curlproc, CURLOPT_POST, true);
- curl_setopt($curlproc, CURLOPT_POSTFIELDS, $get);
-
- }elseif($get !== []){
-
- $get = http_build_query($get);
- $url .= "?" . $get;
- }
-
- curl_setopt($curlproc, CURLOPT_URL, $url);
-
-
- curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-
- curl_setopt($curlproc, CURLOPT_ENCODING, "");
-
- if($is_xhr === true){
-
- curl_setopt($curlproc, CURLOPT_HTTPHEADER,
- ["User-Agent: " . config::USER_AGENT,
- "Accept: application/json",
- "Accept-Language: en-US,en;q=0.5",
- "Accept-Encoding: gzip",
- "Referer: https://www.startpage.com/",
- "Content-Type: application/json",
- "Content-Length: " . strlen($get),
- "Origin: https://www.startpage.com/",
- "DNT: 1",
- "Connection: keep-alive",
- "Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
- "Sec-Fetch-Dest: empty",
- "Sec-Fetch-Mode: cors",
- "Sec-Fetch-Site: same-origin",
- "TE: trailers"]
- );
-
- }elseif($post === true){
-
- curl_setopt($curlproc, CURLOPT_HTTPHEADER,
- ["User-Agent: " . config::USER_AGENT,
- "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
- "Accept-Language: en-US,en;q=0.5",
- "Accept-Encoding: gzip",
- "Referer: https://www.startpage.com/",
- "Content-Type: application/x-www-form-urlencoded",
- "Content-Length: " . strlen($get),
- "DNT: 1",
- "Connection: keep-alive",
- "Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
- "Upgrade-Insecure-Requests: 1",
- "Sec-Fetch-Dest: document",
- "Sec-Fetch-Mode: navigate",
- "Sec-Fetch-Site: none",
- "Sec-Fetch-User: ?1",
- "Priority: u=0, i",
- "TE: trailers"]
- );
- }else{
-
- curl_setopt($curlproc, CURLOPT_HTTPHEADER,
- ["User-Agent: " . config::USER_AGENT,
- "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
- "Accept-Language: en-US,en;q=0.5",
- "Accept-Encoding: gzip",
- "DNT: 1",
- "Connection: keep-alive",
- "Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
- "Sec-Fetch-Dest: document",
- "Sec-Fetch-Mode: navigate",
- "Sec-Fetch-Site: none",
- "Sec-Fetch-User: ?1",
- "Priority: u=0, i",
- "TE: trailers"]
- );
- }
-
- curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
- curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
- curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
- curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
- curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-
- $this->backend->assign_proxy($curlproc, $proxy);
-
- $data = curl_exec($curlproc);
-
- if(curl_errno($curlproc)){
-
- throw new Exception(curl_error($curlproc));
- }
-
- curl_close($curlproc);
- return $data;
- }
-
- public function web($get){
-
- if($get["npt"]){
-
- [$post, $proxy] = $this->backend->get($get["npt"], "web");
-
- try{
- $html = $this->get(
- $proxy,
- "https://www.startpage.com/sp/search",
- $post,
- true
- );
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
-
- $get_instant_answer = false;
-
- }else{
-
- $proxy = $this->backend->get_ip();
-
- $params = [
- "query" => $get["s"],
- "cat" => "web",
- "pl" => "opensearch"
- ];
-
- if($get["nsfw"] == "no"){
-
- $params["qadf"] = "heavy";
- $get_instant_answer = false;
- }else{
-
- $get_instant_answer = true;
- }
-
- if($get["country"] !== "any"){
-
- $params["qsr"] = $get["country"];
- }
-
- if($get["time"] !== "any"){
-
- $params["with_date"] = $get["time"];
- }
-
- try{
- $html = $this->get(
- $proxy,
- "https://www.startpage.com/sp/search",
- $params
- );
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
-
-
- }
-
- $this->detect_captcha($html);
-
- if(
- preg_match(
- '/React\.createElement\(UIStartpage\.AppSerpWeb, ?(.+)\),?$/m',
- $html,
- $matches
- ) === 0
- ){
-
- throw new Exception("Failed to grep JSON object");
- }
-
- $json = json_decode($matches[1], true);
-
- if($json === null){
-
- throw new Exception("Failed to decode JSON");
- }
-
-
-
- $out = [
- "status" => "ok",
- "spelling" => [
- "type" => "no_correction",
- "using" => null,
- "correction" => null
- ],
- "npt" => null,
- "answer" => [],
- "web" => [],
- "image" => [],
- "video" => [],
- "news" => [],
- "related" => []
- ];
-
-
- $out["npt"] = $this->parse_npt($json, "web", $proxy);
-
- foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
-
- if(!isset($category["display_type"])){
-
- continue;
- }
-
- switch($category["display_type"]){
-
- case "web-google":
- foreach($category["results"] as $result){
-
- $sublinks = [];
-
- foreach($result["siteLinks"] as $sublink){
-
- $sublinks[] = [
- "title" => $sublink["title"],
- "description" => null,
- "url" => $sublink["clickUrl"]
- ];
- }
-
- $description =
- explode(
- "...",
- $this->titledots(
- html_entity_decode(
- $this->fuckhtml
- ->getTextContent(
- $result["description"]
- )
- )
- ),
- 2
- );
-
- $date = strtotime(trim($description[0]));
-
- if(
- $date === false ||
- count($description) !== 2 ||
- strlen($description[0]) > 14
- ){
-
-
- $description =
- implode(
- " ... ",
- $description
- );
-
- $date = null;
- }else{
-
-
- $description = ltrim($description[1]);
- }
-
- $out["web"][] = [
- "title" =>
- $this->titledots(
- html_entity_decode(
- $this->fuckhtml
- ->getTextContent(
- $result["title"]
- )
- )
- ),
- "description" => $description,
- "url" => $result["clickUrl"],
- "date" => $date,
- "type" => "web",
- "thumb" => [
- "url" => null,
- "ratio" => null
- ],
- "sublink" => $sublinks,
- "table" => []
- ];
- }
- break;
-
- case "images-qi-top":
- foreach($category["results"] as $result){
-
- $out["image"][] = [
- "title" =>
- $this->titledots(
- html_entity_decode(
- $this->fuckhtml
- ->getTextContent(
- $result["title"]
- )
- )
- ),
- "source" => [
- [
- "url" => $result["rawImageUrl"],
- "width" => (int)$result["width"],
- "height" => (int)$result["height"]
- ],
- [
- "url" => $this->unshitimage($result["mdThumbnailUrl"]),
- "width" => (int)$result["mdThumbnailWidth"],
- "height" => (int)$result["mdThumbnailHeight"]
- ]
- ],
- "url" =>
- $result["altClickUrl"]
- ];
- }
- break;
-
- case "spellsuggest-google":
- $out["spelling"] =
- [
- "type" => "including",
- "using" => $json["render"]["query"],
- "correction" => $category["results"][0]["query"]
- ];
- break;
-
- case "dictionary-qi":
- foreach($category["results"] as $result){
-
- $answer = [
- "title" => $result["word"],
- "description" => [],
- "url" => null,
- "thumb" => null,
- "table" => [],
- "sublink" => []
- ];
-
- foreach($result["lexical_categories"] as $lexic_type => $definitions){
-
- $answer["description"][] = [
- "type" => "title",
- "value" => $lexic_type
- ];
-
- $i = 0;
-
- foreach($definitions as $definition){
-
- $text_definition = trim($definition["definition"]);
- $text_example = trim($definition["example"]);
- $text_synonyms = implode(", ", $definition["synonyms"]);
-
- if($text_definition != ""){
-
- $i++;
-
- $c = count($answer["description"]) - 1;
- if(
- $c !== 0 &&
- $answer["description"][$c]["type"] == "text"
- ){
-
- $answer["description"][$c]["value"] .=
- "\n\n" . $i . ". " . $text_definition;
-
- }else{
-
- $answer["description"][] = [
- "type" => "text",
- "value" => $i . ". " . $text_definition
- ];
- }
- }
-
- if($text_example != ""){
-
- $answer["description"][] = [
- "type" => "quote",
- "value" => $text_example
- ];
- }
-
- if($text_synonyms != ""){
-
- $answer["description"][] = [
- "type" => "text",
- "value" => "Synonyms: " . $text_synonyms
- ];
- }
- }
- }
-
- $out["answer"][] = $answer;
- }
- break;
- }
- }
-
-
- if(
- $get["extendedsearch"] == "yes" &&
- $get_instant_answer === true
- ){
-
-
- try{
- $post = [
- "se" => "n0vze2y9dqwy",
- "q" => $json["render"]["query"],
- "results" => [],
- "enableKnowledgePanel" => true,
- "enableMediaThumbBar" => false,
- "enableSearchSuggestions" => false,
- "enableTripadvisorProperties" => [],
- "enableTripadvisorPlaces" => [],
- "enableTripadvisorPlacesForLocations" => [],
- "enableWebProducts" => false,
- "tripadvisorPartnerId" => null,
- "tripadvisorMapColorMode" => "light",
- "tripadvisorDisablesKnowledgePanel" => false,
- "instantAnswers" => [
- "smartAnswers",
- "youtube",
- "tripadvisor"
- ],
- "iaType" => null,
- "forceEnhancedKnowledgePanel" => false,
- "shoppingOnly" => false,
- "allowAdultProducts" => true,
- "lang" => "en",
- "browserLang" => "en-US",
- "browserTimezone" => "America/New_York",
- "market" => null,
- "userLocation" => null,
- "userDate" => date("Y-m-d"),
- "userAgentType" => "unknown"
- ];
-
- foreach($out["web"] as $result){
-
- $post["results"][] = [
- "url" => $result["url"],
- "title" => $result["title"]
- ];
- }
-
- $post = json_encode($post, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE);
-
- $additional_data =
- $this->get(
- $proxy,
- "https://www.startpage.com/sp/qi?qimsn=ex&sxap=%2Fv1%2Fquery&sc=" . $json["render"]["callback_sc"] . "&sr=1",
- $post,
- true,
- true
- );
-
- $additional_data = json_decode($additional_data, true);
-
- if($additional_data === null){
-
- throw new Exception("Failed to decode JSON");
- }
-
- if(!isset($additional_data["knowledgePanel"])){
-
- throw new Exception("Response has missing data (knowledgePanel)");
- }
-
- $additional_data = $additional_data["knowledgePanel"];
-
- $answer = [
- "title" => $additional_data["meta"]["title"],
- "description" => [
- [
- "type" => "quote",
- "value" => $additional_data["meta"]["description"]
- ]
- ],
- "url" => $additional_data["meta"]["origWikiUrl"],
- "thumb" => $additional_data["meta"]["image"],
- "table" => [],
- "sublink" => []
- ];
-
-
- $this->fuckhtml->load($additional_data["html"]);
-
- $div =
- $this->fuckhtml
- ->getElementsByTagName(
- "div"
- );
-
-
- $description =
- $this->fuckhtml
- ->getElementsByClassName(
- "sx-kp-short-extract sx-kp-short-extract-complete",
- $div
- );
-
- if(count($description) !== 0){
-
- $answer["description"][] = [
- "type" => "text",
- "value" =>
- html_entity_decode(
- $this->fuckhtml
- ->getTextContent(
- $description[0]
- )
- )
- ];
- }
-
-
- $socials =
- $this->fuckhtml
- ->getElementsByClassName(
- "sx-wiki-social-link",
- "a"
- );
-
- foreach($socials as $social){
-
- $title =
- $this->fuckhtml
- ->getTextContent(
- $social["attributes"]["title"]
- );
-
- $url =
- $this->fuckhtml
- ->getTextContent(
- $social["attributes"]["href"]
- );
-
- switch($title){
-
- case "Official Website":
- $title = "Website";
- break;
- }
-
- $answer["sublink"][$title] = $url;
- }
-
-
- $videos =
- $this->fuckhtml
- ->getElementsByClassName(
- "sx-kp-video-grid-item",
- $div
- );
-
- foreach($videos as $video){
-
- $this->fuckhtml->load($video);
-
- $as =
- $this->fuckhtml
- ->getElementsByTagName(
- "a"
- );
-
- if(count($as) === 0){
-
-
- continue;
- }
-
- $image =
- $this->fuckhtml
- ->getElementsByAttributeName(
- "data-sx-src",
- "img"
- );
-
- if(count($image) !== 0){
-
- $thumb = [
- "ratio" => "16:9",
- "url" =>
- $this->fuckhtml
- ->getTextContent(
- $image[0]["attributes"]["data-sx-src"]
- )
- ];
- }else{
- $thumb = [
- "ratio" => null,
- "url" => null
- ];
- }
-
- $out["video"][] = [
- "title" =>
- $this->fuckhtml
- ->getTextContent(
- $as[0]["attributes"]["title"]
- ),
- "description" => null,
- "date" => null,
- "duration" => null,
- "views" => null,
- "thumb" => $thumb,
- "url" =>
- $this->fuckhtml
- ->getTextContent(
- $as[0]["attributes"]["href"]
- )
- ];
- }
-
-
- $this->fuckhtml->load($additional_data["html"]);
-
-
- $table =
- $this->fuckhtml
- ->getElementsByClassName(
- "sx-infobox",
- "table"
- );
-
- if(count($table) !== 0){
-
- $trs =
- $this->fuckhtml
- ->getElementsByTagName(
- "tr"
- );
-
- foreach($trs as $tr){
-
- $this->fuckhtml->load($tr);
-
-
-
-
- $tds =
- $this->fuckhtml
- ->getElementsByTagName(
- "td"
- );
-
- $ths =
- $this->fuckhtml
- ->getElementsByTagName(
- "th"
- );
-
- if(
- count($ths) === 1 &&
- count($tds) === 1
- ){
-
- $title =
- $this->fuckhtml
- ->getTextContent(
- $ths[0]
- );
-
- $description = [];
-
- $this->fuckhtml->load($tds[0]);
-
- $lis =
- $this->fuckhtml
- ->getElementsByTagName(
- "li"
- );
-
- if(count($lis) !== 0){
-
- foreach($lis as $li){
-
- $description[] =
- $this->fuckhtml
- ->getTextContent(
- $li
- );
- }
-
- $description = implode(", ", $description);
- }else{
-
- $description =
- $this->fuckhtml
- ->getTextContent(
- $tds[0]
- );
- }
-
- $answer["table"][$title] = $description;
- }
- }
- }
-
- $out["answer"][] = $answer;
-
- }catch(Exception $error){
-
-
-
- }
- }
-
- return $out;
- }
-
- public function image($get){
-
- if($get["npt"]){
-
- [$post, $proxy] = $this->backend->get($get["npt"], "images");
-
- try{
- $html = $this->get(
- $proxy,
- "https://www.startpage.com/sp/search",
- $post,
- true
- );
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
-
- }else{
-
- $search = $get["s"];
- if(strlen($search) === 0){
-
- throw new Exception("Search term is empty!");
- }
-
- try{
-
- $proxy = $this->backend->get_ip();
-
- $params = [
- "query" => $get["s"],
- "cat" => "images",
- "pl" => "opensearch"
- ];
-
- if($get["nsfw"] == "no"){
-
- $params["qadf"] = "heavy";
- }
-
- if($get["size"] != "any"){
-
- if(
- $get["size"] == "Small" ||
- $get["size"] == "Medium" ||
- $get["size"] == "Large" ||
- $get["size"] == "Wallpaper"
- ){
-
- $params["flimgsize"] = $get["size"];
- }else{
-
- $params["image-size-select"] = "isz:lt,islt:" . $get["size"];
- }
- }
-
- if($get["color"] != "any"){
-
- if($get["color"] == "color"){
-
- $params["flimgcolor"] = "ic:color";
- }elseif($get["color"] == "bnw"){
-
- $params["flimgcolor"] = "ic:gray";
- }else{
-
- $params["flimgcolor"] = "ic:specific,isc:" . $get["color"];
- }
- }
-
- if($get["type"] != "any"){
-
- $params["flimgtype"] = $get["type"];
- }
-
- if($get["license"] != "any"){
-
- $params["flimglicense"] = $get["license"];
- }
-
- try{
- $html = $this->get(
- $proxy,
- "https://www.startpage.com/sp/search",
- $params
- );
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
-
-
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
- }
-
- $this->detect_captcha($html);
-
- $out = [
- "status" => "ok",
- "npt" => null,
- "image" => []
- ];
-
- if(
- preg_match(
- '/React\.createElement\(UIStartpage\.AppSerpImages, ?(.+)\),?$/m',
- $html,
- $matches
- ) === 0
- ){
-
- throw new Exception("Failed to grep JSON object");
- }
-
- $json = json_decode($matches[1], true);
-
- if($json === null){
-
- throw new Exception("Failed to decode JSON object");
- }
-
-
- $out["npt"] = $this->parse_npt($json, "images", $proxy);
-
-
- foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
-
- if($category["display_type"] != "images-bing"){
-
-
- continue;
- }
-
- foreach($category["results"] as $image){
-
- $out["image"][] = [
- "title" => $this->titledots($image["title"]),
- "source" => [
- [
- "url" => $this->unshitimage($image["clickUrl"]),
- "width" => (int)$image["width"],
- "height" => (int)$image["height"]
- ],
- [
- "url" => $this->unshitimage($image["thumbnailUrl"]),
- "width" => (int)$image["thumbnailWidth"],
- "height" => (int)$image["thumbnailHeight"]
- ]
- ],
- "url" => $image["altClickUrl"]
- ];
- }
- }
-
- return $out;
- }
-
- public function video($get){
-
- if($get["npt"]){
-
- [$post, $proxy] = $this->backend->get($get["npt"], "videos");
-
- try{
- $html = $this->get(
- $proxy,
- "https://www.startpage.com/sp/search",
- $post,
- true
- );
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
-
- }else{
-
- $search = $get["s"];
- if(strlen($search) === 0){
-
- throw new Exception("Search term is empty!");
- }
-
- try{
-
- $proxy = $this->backend->get_ip();
-
- $params = [
- "query" => $get["s"],
- "cat" => "video",
- "pl" => "opensearch"
- ];
-
- if($get["nsfw"] == "no"){
-
- $params["qadf"] = "heavy";
- }
-
- if($get["sort"] != "relevance"){
-
- $params["sort_by"] = $get["sort"];
- }
-
- if($get["duration"] != "any"){
-
- $params["with_duration"] = $get["duration"];
- }
-
- try{
- $html = $this->get(
- $proxy,
- "https://www.startpage.com/sp/search",
- $params
- );
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
-
-
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
- }
-
- $this->detect_captcha($html);
-
- if(
- preg_match(
- '/React\.createElement\(UIStartpage\.AppSerpVideos, ?(.+)\),?$/m',
- $html,
- $matches
- ) === 0
- ){
-
- throw new Exception("Failed to get JSON object");
- }
-
- $json = json_decode($matches[1], true);
-
- if($json === null){
-
- throw new Exception("Failed to decode JSON object");
- }
-
- $out = [
- "status" => "ok",
- "npt" => null,
- "video" => [],
- "author" => [],
- "livestream" => [],
- "playlist" => [],
- "reel" => []
- ];
-
-
- $out["npt"] = $this->parse_npt($json, "video", $proxy);
-
-
- foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
-
- if($category["display_type"] == "video-youtube"){
-
- foreach($category["results"] as $video){
-
- if(
- isset($video["thumbnailUrl"]) &&
- $video["thumbnailUrl"] !== null
- ){
-
- $thumb = [
- "ratio" => "16:9",
- "url" => $this->unshitimage($video["thumbnailUrl"])
- ];
- }else{
-
- $thumb = [
- "ratio" => null,
- "url" => null
- ];
- }
-
- $out["video"][] = [
- "title" => $video["title"],
- "description" => $this->limitstrlen($video["description"]),
- "author" => [
- "name" => $video["channelTitle"],
- "url" => null,
- "avatar" => null
- ],
- "date" => strtotime($video["publishDate"]),
- "duration" => $this->hms2int($video["duration"]),
- "views" => (int)$video["viewCount"],
- "thumb" => $thumb,
- "url" => $video["clickUrl"]
- ];
- }
- }
- }
-
- return $out;
- }
-
- public function news($get){
-
- if($get["npt"]){
-
- [$post, $proxy] = $this->backend->get($get["npt"], "news");
-
- try{
- $html = $this->get(
- $proxy,
- "https://www.startpage.com/sp/search",
- $post,
- true
- );
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
-
- }else{
-
- $search = $get["s"];
- if(strlen($search) === 0){
-
- throw new Exception("Search term is empty!");
- }
-
- try{
-
- $proxy = $this->backend->get_ip();
-
- $params = [
- "query" => $get["s"],
- "cat" => "news",
- "pl" => "opensearch"
- ];
-
- if($get["nsfw"] == "no"){
-
- $params["qadf"] = "heavy";
- }
-
- if($get["time"] != "any"){
-
- $params["with_date"] = $get["time"];
- }
-
- try{
- $html = $this->get(
- $proxy,
- "https://www.startpage.com/sp/search",
- $params
- );
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
-
-
- }catch(Exception $error){
-
- throw new Exception("Failed to fetch search page");
- }
- }
-
- $this->detect_captcha($html);
-
- if(
- preg_match(
- '/React\.createElement\(UIStartpage\.AppSerpNews, ?(.+)\),?$/m',
- $html,
- $matches
- ) === 0
- ){
-
- throw new Exception("Failed to get JSON object");
- }
-
- $json = json_decode($matches[1], true);
-
- if($json === null){
-
- throw new Exception("Failed to decode JSON object");
- }
-
- $out = [
- "status" => "ok",
- "npt" => null,
- "news" => []
- ];
-
-
- $out["npt"] = $this->parse_npt($json, "news", $proxy);
-
- foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
-
- if($category["display_type"] != "news-bing"){
-
-
- continue;
- }
-
- foreach($category["results"] as $news){
-
- if(
- isset($news["thumbnailUrl"]) &&
- $news["thumbnailUrl"] !== null
- ){
-
- $thumb = [
- "ratio" => "16:9",
- "url" => $this->unshitimage($news["thumbnailUrl"])
- ];
- }else{
-
- $thumb = [
- "ratio" => null,
- "url" => null
- ];
- }
-
- $out["news"][] = [
- "title" => $this->titledots($this->remove_penguins($news["title"])),
- "author" => $news["source"],
- "description" => $this->titledots($this->remove_penguins($news["description"])),
- "date" => (int)substr((string)$news["date"], 0, -3),
- "thumb" => $thumb,
- "url" => $news["clickUrl"]
- ];
- }
- }
-
- return $out;
- }
-
- private function parse_npt($json, $pagetype, $proxy){
-
- foreach($json["render"]["presenter"]["pagination"]["pages"] as $page){
-
- if($page["name"] == "Next"){
-
- parse_str(
- explode(
- "?",
- $page["url"],
- 2
- )[1],
- $str
- );
-
- return
- $this->backend->store(
- http_build_query(
- [
- "lui" => "english",
- "language" => "english",
- "query" => $str["q"],
- "cat" => $pagetype,
- "sc" => $str["sc"],
- "t" => "device",
- "segment" => "startpage.udog",
- "page" => $str["page"]
- ]
- ),
- $pagetype,
- $proxy
- );
-
- break;
- }
- }
-
- return null;
- }
-
- private function unshitimage($url){
-
- $query = parse_url($url, PHP_URL_QUERY);
- parse_str($query, $query);
-
- if(isset($query["piurl"])){
-
- if(strpos($query["piurl"], "gstatic.com/")){
-
- return
- explode(
- "&",
- $query["piurl"],
- 2
- )[0];
- }
-
- if(
- strpos($query["piurl"], "bing.net/") ||
- strpos($query["piurl"], "bing.com/")
- ){
-
- return
- explode(
- "&",
- $query["piurl"],
- 2
- )[0];
- }
-
- return $query["piurl"];
- }
-
- return $url;
- }
-
- private function limitstrlen($text){
-
- return
- explode(
- "\n",
- wordwrap(
- str_replace(
- ["\n\r", "\r\n", "\n", "\r"],
- " ",
- $text
- ),
- 300,
- "\n"
- ),
- 2
- )[0];
- }
-
- private function titledots($title){
-
- return trim($title, " .\t\n\r\0\x0B…");
- }
-
- private function hms2int($time){
-
- $parts = explode(":", $time, 3);
- $time = 0;
-
- if(count($parts) === 3){
-
-
- $time = $time + ((int)$parts[0] * 3600);
- array_shift($parts);
- }
-
- if(count($parts) === 2){
-
-
- $time = $time + ((int)$parts[0] * 60);
- array_shift($parts);
- }
-
-
- $time = $time + (int)$parts[0];
-
- return $time;
- }
-
- private function remove_penguins($text){
-
- return str_replace(
- ["", ""],
- "",
- $text
- );
- }
-
- private function detect_captcha($html){
-
- $this->fuckhtml->load($html);
-
- $title =
- $this->fuckhtml
- ->getElementsByTagName(
- "title"
- );
-
- if(
- count($title) !== 0 &&
- $title[0]["innerHTML"] == "Redirecting..."
- ){
-
-
- $as =
- $this->fuckhtml
- ->getElementsByTagName(
- "a"
- );
-
- foreach($as as $a){
-
- if(
- strpos(
- $this->fuckhtml
- ->getTextContent(
- $a["innerHTML"]
- ),
- "https://www.startpage.com/sp/captcha"
- ) !== false
- ){
-
- throw new Exception("Startpage returned a captcha");
- }
- }
-
- throw new Exception("Startpage redirected the scraper to an unhandled page");
- }
- }
- }
|