fix local https oauth

.gitignore

@@ -0,0 +1 @@
+vendor

classes/Common.php

@@ -30,8 +30,7 @@ final class Common {
 	private $browser_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:57.0) Gecko/20100101 Firefox/57.0" ;
 	private	$db_servers = [
 			'fast' => '.web.db.svc.eqiad.wmflabs' ,
-			'slow' => '.analytics.db.svc.eqiad.wmflabs' ,
-			'old' => '.labsdb'
+			'slow' => '.analytics.db.svc.eqiad.wmflabs'
 		] ;
 
 	private $cookiejar ; # For doPostRequest
@@ -152,7 +151,7 @@ function getDBname ( $language , $project ) /*:string*/ {
 		elseif ( $project == 'wikispecies' ) $ret = 'specieswiki_p' ;
 		elseif ( $language == 'meta' ) $ret .= 'metawiki_p' ;
 		else if ( $project == 'wikimedia' ) $ret .= $language.$project."_p" ;
-		else die ( "Cannot construct database name for $language.$project - aborting." ) ;
+		else throw new \Exception ( "Cannot construct database name for $language.$project - aborting." ) ;
 		return $ret ;
 	}
 
@@ -215,13 +214,6 @@ public function openDB ( $language , $project , $slow_queries = false , $persist
 			$db = @new mysqli($server, $this->mysql_user, $this->mysql_password , $dbname);
 		}
 
-		# Try the old server as fallback
-		if($db->connect_errno > 0) {
-			$server = substr( $dbname, 0, -2 ) . $this->db_servers['old'];
-			if ( $persistent ) $server = "p:$server" ;
-			$db = @new mysqli($server, $this->mysql_user, $this->mysql_password , $dbname);
-		}
-
 		assert ( $db->connect_errno == 0 , 'Unable to connect to database [' . $db->connect_error . ']' ) ;
 		if ( !$persistent and $this->use_db_cache ) $this->db_cache[$db_key] = $db ;
 		return $db ;
@@ -244,7 +236,6 @@ public function getSQL ( &$db , &$sql , $max_tries = 5 , $message = '' ) {
 		var_dump($sql);
 		var_dump($e->getTraceAsString());
 		throw $e;
-#		die ( 'There was an error running the query [' . $db->error . '/' . $db->errno . ']'."\n$sql\n$message\n" ) ;
 	}
 
 	public function findSubcats ( &$db , $root , &$subcats , $depth = -1 ) {
@@ -427,7 +418,7 @@ public function getSPARQL ( $cmd ) {
 			$again = preg_match ( '/429/' , $http_response_header[0] ) ;
 			if ( $again ) {
 				$cnt++ ;
-				if ( $cnt > $max ) die ( "SPARQL wait too long ({$cnt}x):\n{$sparql}\n") ;
+				if ( $cnt > $max ) throw new \Exception ( "SPARQL wait too long ({$cnt}x):\n{$sparql}\n") ;
 				sleep ( 5 ) ;
 			}
 		} while ( $again ) ;
@@ -463,13 +454,55 @@ public function getSPARQLitems ( $cmd , $varname = '' ) {
 		return $ret ;
 	}
 
+	public function getSPARQL_TSV ( $query ) {
+		$query = "$query\n#TOOL: {$this->toolname}" ;
+		set_time_limit(0);
+
+		$url = "https://query.wikidata.org/sparql?query=".urlencode($query) ;
+		$headers = ['Accept: text/csv'] ;
+		$agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:56.0) Gecko/20100101 Firefox/56.0';
+
+		$fh = tmpfile();
+		$callback = function ($ch, $data) use ($fh){ return fwrite($fh, $data); } ;
+
+		$ch = curl_init();
+		curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
+		curl_setopt($ch, CURLOPT_VERBOSE, false);
+		curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($ch, CURLOPT_USERAGENT, $agent);
+		curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
+		curl_setopt($ch, CURLOPT_WRITEFUNCTION, $callback);
+		curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
+		curl_setopt($ch, CURLOPT_URL,$url);
+		curl_exec($ch);
+		curl_close($ch);
+
+		# Flush and reset file position to start
+		fflush($fh);
+		fseek($fh,0);
+
+		# Read and yield
+		$header = [] ;
+		while (($data = fgetcsv($fh, 10000)) !== FALSE) {
+			if ( count($header)==0 ) {
+				$header = $data ;
+				continue ;
+			}
+			$o = [] ;
+			foreach ( $data AS $col => $s ) $o[$header[$col]] = $s ;
+			yield $o ;
+		}
+		fclose($fh);
+		yield from [];
+	}
+
 // QuickStatements
 // require_once ( '/data/project/quickstatements/public_html/quickstatements.php' ) ;
 
 	// Will use the first *.conf config file in tools directory, unless specified
 	public function getQS ( $toolname , $config_file = '' , $useTemporaryBatchID = false ) {
 		if ( $config_file == '' ) $config_file = $this->guessConfigFile() ;
-		if ( $config_file == '' ) die ( "Can't determine QS config file location" ) ;
+		if ( $config_file == '' ) throw new \Exception ( "Can't determine QS config file location" ) ;
 		$qs = new QuickStatements() ;
 		$qs->use_oauth = false ;
 		$qs->bot_config_file = $config_file ;

classes/OAuth.php

@@ -15,7 +15,8 @@ class OAuth {
 	var $language , $project ;
 	var $ini_file , $params ;
 	var $cookie_lifetime;
-	var $mwOAuthUrl = 'https://www.mediawiki.org/w/index.php?title=Special:OAuth';
+	#var $mwOAuthUrl = 'https://www.mediawiki.org/w/index.php?title=Special:OAuth'; # See https://phabricator.wikimedia.org/T112730
+	var $mwOAuthUrl = 'https://www.mediawiki.org/wiki/Special:OAuth';
 	var $publicMwOAuthUrl; //if the mediawiki url given to the user is different from how this
 							//script may see it (e.g. if behind a proxy) set the user url here.
 	var $mwOAuthIW = 'mw'; // Set this to the interwiki prefix for the OAuth central wiki.
@@ -25,6 +26,7 @@ class OAuth {
 	var $delay_after_create_s = 2 ;
 	var $delay_after_edit_s = 1 ;
 	var $delay_after_upload_s = 1 ;
+	var $cookie_best_before = 60*60*24*30*3 ;# expires in three months
 
 	function __construct ( $t , $l = '' , $p = '' , $oauth_url = '', $c = null ) {
 		if ( is_array($t) ) { // Bespoke override for third-party sites
@@ -78,11 +80,15 @@ function sleepAfterEdit ( $type ) {
 		if ( $type == 'upload' ) sleep ( $this->delay_after_upload_s ) ;
 	}
 
+	function getCookiePath() {
+		return '/'.$this->tool.'/'; // needed for wbstack
+	}
+
 	function logout () {
 		$this->setupSession() ;
 		session_start();
-		setcookie ( 'tokenKey' , '' , 1 , '/'.$this->tool.'/' ) ;
-		setcookie ( 'tokenSecret' , '' , 1 , '/'.$this->tool.'/' ) ;
+		setcookie ( 'tokenKey' , '' , 1 , $this->getCookiePath() ) ;
+		setcookie ( 'tokenSecret' , '' , 1 , $this->getCookiePath() ) ;
 		$_SESSION['tokenKey'] = '' ;
 		$_SESSION['tokenSecret'] = '' ;
 		session_write_close();
@@ -128,14 +134,19 @@ function loadToken() {
 		session_write_close();
 	}
 
+	function getOAuthURL ( $action , $append_separator = false ) {
+		$url = $this->mwOAuthUrl . '/' . $action ;
+		if ( $append_separator ) $url .= strpos( $url, '?' ) ? '&' : '?';
+		return $url ;
+	}
+
 
 	/**
 	 * Handle a callback to fetch the access token
 	 * @return void
 	 */
 	function fetchAccessToken() {
-		$url = $this->mwOAuthUrl . '/token';
-		$url .= strpos( $url, '?' ) ? '&' : '?';
+		$url = $this->getOAuthURL ( 'token' , true ) ;
 		$url .= http_build_query( [
 			'format' => 'json',
 			'oauth_verifier' => $_GET['oauth_verifier'],
@@ -188,10 +199,11 @@ function fetchAccessToken() {
 			if ( $this->cookie_lifetime !== null ) {
                 $t += $this->cookie_lifetime;
 			} else {
-                $t += 60*60*24*30*3 ; // expires in three months
+				$t += $this->cookie_best_before ;
             }
-			setcookie ( 'tokenKey' , $_SESSION['tokenKey'] , $t , '/'.$this->tool.'/' ) ;
-			setcookie ( 'tokenSecret' , $_SESSION['tokenSecret'] , $t , '/'.$this->tool.'/' ) ;
+
+			setcookie ( 'tokenKey' , $_SESSION['tokenKey'] , $t , $this->getCookiePath() ) ;
+			setcookie ( 'tokenSecret' , $_SESSION['tokenSecret'] , $t , $this->getCookiePath() ) ;
 		}
 		session_write_close();
 	}
@@ -258,8 +270,7 @@ function doAuthorizationRedirect($callback='') {
 		// First, we need to fetch a request token.
 		// The request is signed with an empty token secret and no token key.
 		$this->gTokenSecret = '';
-		$url = $this->mwOAuthUrl . '/initiate';
-		$url .= strpos( $url, '?' ) ? '&' : '?';
+		$url = $this->getOAuthURL ( 'initiate' , true ) ;
 		$query = [
 			'format' => 'json',
 
@@ -315,14 +326,13 @@ function doAuthorizationRedirect($callback='') {
                 $t += 60*60*24*30; // expires in one month
             }
 
-			setcookie ( 'tokenKey' , $_SESSION['tokenKey'] , $t , '/'.$this->tool.'/' ) ;
-			setcookie ( 'tokenSecret' , $_SESSION['tokenSecret'] , $t , '/'.$this->tool.'/' ) ;
+			setcookie ( 'tokenKey' , $_SESSION['tokenKey'] , $t , $this->getCookiePath() ) ;
+			setcookie ( 'tokenSecret' , $_SESSION['tokenSecret'] , $t , $this->getCookiePath() ) ;
 		}
 		session_write_close();
 
 		// Then we send the user off to authorize
-		$url = $this->publicMwOAuthUrl . '/authorize';
-		$url .= strpos( $url, '?' ) ? '&' : '?';
+		$url = $this->getOAuthURL ( 'authorize' , true ) ;
 		$arr = [
 			'oauth_token' => $token->key,
 			'oauth_consumer_key' => $this->gConsumerKey,
@@ -336,7 +346,7 @@ function doAuthorizationRedirect($callback='') {
 
 	function doIdentify() {
 
-		$url = $this->mwOAuthUrl . '/identify';
+		$url = $this->getOAuthURL ( 'identify' , false ) ;
 		$headerArr = [
 			// OAuth information
 			'oauth_consumer_key' => $this->gConsumerKey,
@@ -708,9 +718,9 @@ function setSitelink ( $q , $site , $title ) {
 
 		return true ;
 	}
-
-
-	function setDesc ( $q , $text , $language ) {
+	
+	
+	function setDesc ( $q , $text , $language , $summary = '' ) {
 
 		// Fetch the edit token
 		$ch = null;

classes/Widar.php

@@ -54,7 +54,8 @@ public function process_request ( $parameter_name = 'action' ) {
 					explode(',',$this->tfc->getRequest('ids','')),
 					$this->tfc->getRequest('prop',''),
 					$this->tfc->getRequest('target',''),
-					$this->tfc->getRequest('claim','')
+					$this->tfc->getRequest('claim',''),
+					$this->tfc->getRequest('summary','')
 				) ;
 				break ;
 			case 'merge_items':
@@ -390,15 +391,15 @@ public function merge_items ( $from = '' , $to = '' ) {
 		if ( !$this->oa->mergeItems ( $from , $to ) ) throw new \Exception ( "Problem merging item '{$from}' into '{$to}'" ) ;
 	}
 
-	public function set_claims ( $ids , $prop , $target , $qualifier_claim ) {
+	public function set_claims ( $ids , $prop , $target , $qualifier_claim , $summary = '' ) {
 		$this->ensureAuth() ;
 		if ( count($ids) == 0 or $prop == '' or $target == '' ) throw new \Exception ( "set_claim parameters incomplete" ) ;
 		foreach ( $ids AS $id ) {
 			$id = trim ( $id ) ;
 			if ( $id == '' && $qualifier_claim == '' ) continue ;
 			$claim = [ "prop" => $prop , "target" => $target , "type" => "item" ] ;
 			$this->set_q_or_claim ( $claim , $id , $qualifier_claim ) ;
-			if ( !$this->oa->setClaim ( $claim ) ) throw new \Exception ( "set_claims failed: {$id}/{$prop}/{$target}/{$qualifier_claim}" ) ;
+			if ( !$this->oa->setClaim ( $claim , $summary ) ) throw new \Exception ( "set_claims failed: {$id}/{$prop}/{$target}/{$qualifier_claim}" ) ;
 		}
 	}
 

classes/Wikidata.php

@@ -4,7 +4,8 @@
 
 $wikidata_api_url = 'https://www.wikidata.org/w/api.php' ;
 
-require_once ( __DIR__ . '/../../classes/WikidataItem.php' ) ;
+if ( __NAMESPACE__ == 'Toolforge' ) {
+} else require_once ( __DIR__ . '/../../classes/WikidataItem.php' ) ;
 
 class Wikidata {
 
@@ -58,7 +59,11 @@ public function updateItem ( $q ) {
 	protected function parseEntities ( $j ) {
 		foreach ( $j->entities AS $q => $v ) {
 			if ( isset ( $this->items[$q] ) ) continue ; // Paranoia
-			$this->items[$q] = new Toolforge\WikidataItem ;
+			if ( __NAMESPACE__ == 'Toolforge' ) {
+				$this->items[$q] = new WikidataItem ;
+			} else {
+				$this->items[$q] = new Toolforge\WikidataItem ;
+			}
 			$this->items[$q]->q = $q ;
 			$this->items[$q]->j = $v ;
 		}
@@ -173,7 +178,7 @@ protected function getMultipleURLsInParallel ( $urls ) {
 		foreach ( $batches AS $batch_urls ) {
 
 			$mh = curl_multi_init();
-			curl_multi_setopt  ( $mh , CURLMOPT_PIPELINING , 1 ) ;
+			curl_multi_setopt  ( $mh , CURLMOPT_PIPELINING , CURLPIPE_MULTIPLEX ) ;
 			$ch = [] ;
 			foreach ( $batch_urls AS $key => $value ) {
 				$ch[$key] = curl_init($value);

composer.json

@@ -3,7 +3,7 @@
     "description": "Useful classes for Wikimedia Toolforge",
     "homepage": "https://bitbucket.com/magnusmanske/magnustools",
     "license": "GPL-3.0-or-later",
-    "version": "v1.0.5",
+    "version": "v1.0.8",
     "type": "library",
 	"authors": [
         {
@@ -18,10 +18,17 @@
             "Toolforge\\": "classes/"
         }
     },
+    "support":{
+        "issues": "https://bitbucket.org/magnusmanske/magnustools/issues?status=new&status=open",
+        "source": "https://bitbucket.org/magnusmanske/magnustools/"
+    },
     "require-dev": {
         "phpunit/phpunit": "^8.0"
     },
     "require": {
-		"php": ">=7.0.0"
+		"php": ">=7.0.0",
+        "abraham/twitteroauth": "^2.0",
+	"mediawiki/oauthclient": "^1.2",
+        "addwiki/mediawiki-api": "^2.8"
     }
 }