http2 on ddg

scraper/ddg.php

@@ -28,6 +28,9 @@ class ddg{
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
+		// http2 bypass
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		
 		switch($reqtype){
 			case self::req_web:
 				$headers =
@@ -36,27 +39,33 @@ class ddg{
 					"Accept-Encoding: gzip",
 					"Accept-Language: en-US,en;q=0.5",
 					"DNT: 1",
+					"Sec-GPC: 1",
 					"Connection: keep-alive",
 					"Upgrade-Insecure-Requests: 1",
 					"Sec-Fetch-Dest: document",
 					"Sec-Fetch-Mode: navigate",
-					"Sec-Fetch-Site: cross-site",
+					"Sec-Fetch-Site: same-origin",
-					"Upgrade-Insecure-Requests: 1"];
+					"Sec-Fetch-User: ?1",
+					"Priority: u=0, i",
+					"TE: trailers"];
 				break;
 			
 			case self::req_xhr:
 				$headers =
 					["User-Agent: " . config::USER_AGENT,
-					"Accept: */*",
+					"Accept: application/json, text/javascript, */*; q=0.01",
 					"Accept-Encoding: gzip",
 					"Accept-Language: en-US,en;q=0.5",
 					"Connection: keep-alive",
 					"Referer: https://duckduckgo.com/",
 					"X-Requested-With: XMLHttpRequest",
 					"DNT: 1",
-					"Sec-Fetch-Dest: script",
+					"Sec-GPC: 1",
-					"Sec-Fetch-Mode: no-cors",
+					"Connection: keep-alive",
-					"Sec-Fetch-Site: same-site"];
+					"Sec-Fetch-Dest: empty",
+					"Sec-Fetch-Mode: cors",
+					"Sec-Fetch-Site: same-origin",
+					"TE: trailers"];
 				break;
 		}
 		
@@ -1889,12 +1898,12 @@ class ddg{
 			[$npt, $proxy] = $this->backend->get($get["npt"], "images");
 			
 			try{
-				$json = json_decode($this->get(
+				$json = $this->get(
 					$proxy,
 					"https://duckduckgo.com/i.js?" . $npt,
 					[],
 					ddg::req_xhr
-				), true);
+				);
 				
 			}catch(Exception $err){
 				
@@ -1920,6 +1929,7 @@ class ddg{
 			
 			$filter = [];
 			$get_filters = [
+				"hps" => "1",
 				"q" => $search,
 				"iax" => "images",
 				"ia" => "images"
@@ -1970,7 +1980,7 @@ class ddg{
 			}
 			
 			$vqd = $vqd[1];
-				
+			
 			// @TODO: s param = image offset
 			$js_params = [
 				"l" => $country,
@@ -1994,12 +2004,12 @@ class ddg{
 			}
 			
 			try{
-				$json = json_decode($this->get(
+				$json = $this->get(
 					$proxy,
 					"https://duckduckgo.com/i.js",
 					$js_params,
 					ddg::req_xhr
-				), true);
+				);
 				
 			}catch(Exception $err){
 				
@@ -2007,6 +2017,13 @@ class ddg{
 			}
 		}
 		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
 		$out = [
 			"status" => "ok",
 			"npt" => null,

scraper/marginalia.php

@@ -220,6 +220,7 @@ class marginalia{
 			"related" => []
 		];
 		
+		// API scraper
 		if(config::MARGINALIA_API_KEY !== null){
 			
 			try{
@@ -263,34 +264,57 @@ class marginalia{
 			return $out;
 		}
 		
-		// no more cloudflare!! Parse html by default
+		// HTML parser
-		$params = [
+		$proxy = $this->backend->get_ip();
-			"query" => $search
-		];
 		
-		foreach(["adtech", "recent", "intitle"] as $v){
+		if($get["npt"]){
 			
-			if($get[$v] == "yes"){
+			[$params, $proxy] =
+				$this->backend->get(
+					$get["npt"],
+					"web"
+				);
+			
+			try{
+				$html =
+					$this->get(
+						$proxy,
+						"https://search.marginalia.nu/search?" . $params
+					);
+			}catch(Exception $error){
 				
-				switch($v){
+				throw new Exception("Failed to get HTML");
+			}
+			
+		}else{
+			$params = [
+				"query" => $search
+			];
+			
+			foreach(["adtech", "recent", "intitle"] as $v){
+				
+				if($get[$v] == "yes"){
 					
-					case "adtech": $params["adtech"] = "reduce"; break;
+					switch($v){
-					case "recent": $params["recent"] = "recent"; break;
+						
-					case "adtech": $params["searchTitle"] = "title"; break;
+						case "adtech": $params["adtech"] = "reduce"; break;
+						case "recent": $params["recent"] = "recent"; break;
+						case "adtech": $params["searchTitle"] = "title"; break;
+					}
 				}
 			}
-		}
-		
-		try{
-			$html =
-				$this->get(
-					$this->backend->get_ip(),
-					"https://search.marginalia.nu/search",
-					$params
-				);
-		}catch(Exception $error){
 			
-			throw new Exception("Failed to get HTML");
+			try{
+				$html =
+					$this->get(
+						$proxy,
+						"https://search.marginalia.nu/search",
+						$params
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to get HTML");
+			}
 		}
 		
 		$this->fuckhtml->load($html);
@@ -387,6 +411,65 @@ class marginalia{
 			];
 		}
 		
+		// get next page
+		$this->fuckhtml->load($html);
+		
+		$pagination =
+			$this->fuckhtml
+			->getElementsByAttributeValue(
+				"aria-label",
+				"pagination",
+				"nav"
+			);
+		
+		if(count($pagination) === 0){
+			
+			// no pagination
+			return $out;
+		}
+		
+		$this->fuckhtml->load($pagination[0]);
+		
+		$pages =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"page-link",
+				"a"
+			);
+		
+		$found_current_page = false;
+		
+		foreach($pages as $page){
+			
+			if(
+				stripos(
+					$page["attributes"]["class"],
+					"active"
+				) !== false
+			){
+				
+				$found_current_page = true;
+				continue;
+			}
+			
+			if($found_current_page){
+				
+				// we found current page index, and we iterated over
+				// the next page <a>
+				
+				$out["npt"] =
+					$this->backend->store(
+						parse_url(
+							$page["attributes"]["href"],
+							PHP_URL_QUERY
+						),
+						"web",
+						$proxy
+					);
+				break;
+			}
+		}
+		
 		return $out;
 	}
 }

scraper/mojeek.php

@@ -701,9 +701,11 @@ class mojeek{
 				if(count($thumb) === 2){
 					
 					$answer["thumb"] =
-						$this->fuckhtml
+						urldecode(
-						->getTextContent(
+							$this->fuckhtml
-							$thumb[1]
+							->getTextContent(
+								$thumb[1]
+							)
 						);
 				}
 			}