From d92d7acb1112b2d51598ae9a9c93486e8a90e283 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Wed, 23 Jun 2021 20:50:29 +0000 Subject: [PATCH 01/45] buggregator --- public_html/php/ToolforgeCommon.php | 17 +- public_html/php/Widar.php | 7 +- public_html/php/oauth.php | 19 +- public_html/resources/js/flickr2commons.js | 2 +- .../resources/vue/typeahead-search.html | 4 +- public_html/unused_files.php | 9 +- scripts/buggregator/Buggregator.php | 269 ++++++++++++++++++ scripts/buggregator/update.php | 10 + 8 files changed, 308 insertions(+), 29 deletions(-) create mode 100644 scripts/buggregator/Buggregator.php create mode 100755 scripts/buggregator/update.php diff --git a/public_html/php/ToolforgeCommon.php b/public_html/php/ToolforgeCommon.php index 99bb144..fbe7baf 100644 --- a/public_html/php/ToolforgeCommon.php +++ b/public_html/php/ToolforgeCommon.php @@ -28,8 +28,7 @@ final class ToolforgeCommon { private $browser_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:57.0) Gecko/20100101 Firefox/57.0" ; private $db_servers = [ 'fast' => '.web.db.svc.eqiad.wmflabs' , - 'slow' => '.analytics.db.svc.eqiad.wmflabs' , - 'old' => '.labsdb' + 'slow' => '.analytics.db.svc.eqiad.wmflabs' ] ; private $cookiejar ; # For doPostRequest @@ -150,7 +149,7 @@ function getDBname ( $language , $project ) /*:string*/ { elseif ( $project == 'wikispecies' ) $ret = 'specieswiki_p' ; elseif ( $language == 'meta' ) $ret .= 'metawiki_p' ; else if ( $project == 'wikimedia' ) $ret .= $language.$project."_p" ; - else die ( "Cannot construct database name for $language.$project - aborting." ) ; + else throw new \Exception ( "Cannot construct database name for $language.$project - aborting." ) ; return $ret ; } @@ -213,13 +212,6 @@ public function openDB ( $language , $project , $slow_queries = false , $persist $db = @new mysqli($server, $this->mysql_user, $this->mysql_password , $dbname); } - # Try the old server as fallback - if($db->connect_errno > 0) { - $server = substr( $dbname, 0, -2 ) . $this->db_servers['old']; - if ( $persistent ) $server = "p:$server" ; - $db = @new mysqli($server, $this->mysql_user, $this->mysql_password , $dbname); - } - assert ( $db->connect_errno == 0 , 'Unable to connect to database [' . $db->connect_error . ']' ) ; if ( !$persistent and $this->use_db_cache ) $this->db_cache[$db_key] = $db ; return $db ; @@ -242,7 +234,6 @@ public function getSQL ( &$db , &$sql , $max_tries = 5 , $message = '' ) { var_dump($sql); var_dump($e->getTraceAsString()); throw $e; -# die ( 'There was an error running the query [' . $db->error . '/' . $db->errno . ']'."\n$sql\n$message\n" ) ; } public function findSubcats ( &$db , $root , &$subcats , $depth = -1 ) { @@ -422,7 +413,7 @@ public function getSPARQL ( $cmd ) { $again = preg_match ( '/429/' , $http_response_header[0] ) ; if ( $again ) { $cnt++ ; - if ( $cnt > $max ) die ( "SPARQL wait too long ({$cnt}x):\n{$sparql}\n") ; + if ( $cnt > $max ) throw new \Exception ( "SPARQL wait too long ({$cnt}x):\n{$sparql}\n") ; sleep ( 5 ) ; } } while ( $again ) ; @@ -464,7 +455,7 @@ public function getSPARQLitems ( $cmd , $varname = '' ) { // Will use the first *.conf config file in tools directory, unless specified public function getQS ( $toolname , $config_file = '' , $useTemporaryBatchID = false ) { if ( $config_file == '' ) $config_file = $this->guessConfigFile() ; - if ( $config_file == '' ) die ( "Can't determine QS config file location" ) ; + if ( $config_file == '' ) throw new \Exception ( "Can't determine QS config file location" ) ; $qs = new QuickStatements() ; $qs->use_oauth = false ; $qs->bot_config_file = $config_file ; diff --git a/public_html/php/Widar.php b/public_html/php/Widar.php index e55c007..3a8633c 100644 --- a/public_html/php/Widar.php +++ b/public_html/php/Widar.php @@ -54,7 +54,8 @@ public function process_request ( $parameter_name = 'action' ) { explode(',',$this->tfc->getRequest('ids','')), $this->tfc->getRequest('prop',''), $this->tfc->getRequest('target',''), - $this->tfc->getRequest('claim','') + $this->tfc->getRequest('claim',''), + $this->tfc->getRequest('summary','') ) ; break ; case 'merge_items': @@ -390,7 +391,7 @@ public function merge_items ( $from = '' , $to = '' ) { if ( !$this->oa->mergeItems ( $from , $to ) ) throw new Exception ( "Problem merging item '{$from}' into '{$to}'" ) ; } - public function set_claims ( $ids , $prop , $target , $qualifier_claim ) { + public function set_claims ( $ids , $prop , $target , $qualifier_claim , $summary = '' ) { $this->ensureAuth() ; if ( count($ids) == 0 or $prop == '' or $target == '' ) throw new Exception ( "set_claim parameters incomplete" ) ; foreach ( $ids AS $id ) { @@ -398,7 +399,7 @@ public function set_claims ( $ids , $prop , $target , $qualifier_claim ) { if ( $id == '' && $qualifier_claim == '' ) continue ; $claim = [ "prop" => $prop , "target" => $target , "type" => "item" ] ; $this->set_q_or_claim ( $claim , $id , $qualifier_claim ) ; - if ( !$this->oa->setClaim ( $claim ) ) throw new Exception ( "set_claims failed: {$id}/{$prop}/{$target}/{$qualifier_claim}" ) ; + if ( !$this->oa->setClaim ( $claim , $summary ) ) throw new Exception ( "set_claims failed: {$id}/{$prop}/{$target}/{$qualifier_claim}" ) ; } } diff --git a/public_html/php/oauth.php b/public_html/php/oauth.php index f7b87a5..28c76b8 100644 --- a/public_html/php/oauth.php +++ b/public_html/php/oauth.php @@ -20,6 +20,7 @@ class MW_OAuth { var $delay_after_create_s = 2 ; var $delay_after_edit_s = 1 ; var $delay_after_upload_s = 1 ; + var $cookie_best_before = 60*60*24*30*3 ;# expires in three months function __construct ( $t , $l = '' , $p = '' , $oauth_url = '' ) { if ( is_array($t) ) { // Bespoke override for third-party sites @@ -72,12 +73,16 @@ function sleepAfterEdit ( $type ) { if ( $type == 'edit' ) sleep ( $this->delay_after_edit_s ) ; if ( $type == 'upload' ) sleep ( $this->delay_after_upload_s ) ; } + + function getCookiePath() { + return '/' ; # '/'.$this->tool.'/' # Unnecessary with toolname.toolforge.org + } function logout () { $this->setupSession() ; session_start(); - setcookie ( 'tokenKey' , '' , 1 , '/'.$this->tool.'/' ) ; - setcookie ( 'tokenSecret' , '' , 1 , '/'.$this->tool.'/' ) ; + setcookie ( 'tokenKey' , '' , 1 , $this->getCookiePath() ) ; + setcookie ( 'tokenSecret' , '' , 1 , $this->getCookiePath() ) ; $_SESSION['tokenKey'] = '' ; $_SESSION['tokenSecret'] = '' ; session_write_close(); @@ -174,9 +179,9 @@ function fetchAccessToken() { $_SESSION['tokenKey'] = $this->gTokenKey = $token->key; $_SESSION['tokenSecret'] = $this->gTokenSecret = $token->secret; if ( $this->use_cookies ) { - $t = time()+60*60*24*30*3 ; // expires in three months - setcookie ( 'tokenKey' , $_SESSION['tokenKey'] , $t , '/'.$this->tool.'/' ) ; - setcookie ( 'tokenSecret' , $_SESSION['tokenSecret'] , $t , '/'.$this->tool.'/' ) ; + $t = time()+$this->cookie_best_before ; + setcookie ( 'tokenKey' , $_SESSION['tokenKey'] , $t , $this->getCookiePath() ) ; + setcookie ( 'tokenSecret' , $_SESSION['tokenSecret'] , $t , $this->getCookiePath() ) ; } session_write_close(); } @@ -294,8 +299,8 @@ function doAuthorizationRedirect($callback='') { $_SESSION['tokenSecret'] = $token->secret; if ( $this->use_cookies ) { $t = time()+60*60*24*30 ; // expires in one month - setcookie ( 'tokenKey' , $_SESSION['tokenKey'] , $t , '/'.$this->tool.'/' ) ; - setcookie ( 'tokenSecret' , $_SESSION['tokenSecret'] , $t , '/'.$this->tool.'/' ) ; + setcookie ( 'tokenKey' , $_SESSION['tokenKey'] , $t , $this->getCookiePath() ) ; + setcookie ( 'tokenSecret' , $_SESSION['tokenSecret'] , $t , $this->getCookiePath() ) ; } session_write_close(); diff --git a/public_html/resources/js/flickr2commons.js b/public_html/resources/js/flickr2commons.js index e660137..4ee6f9b 100644 --- a/public_html/resources/js/flickr2commons.js +++ b/public_html/resources/js/flickr2commons.js @@ -3,7 +3,7 @@ var flickr2commons = { flinfo: 'https://fist.toolforge.org/flinfo/flinfo.php' , // '/flickr2commons/flinfo_proxy.php' oauth_uploader_base : 'https://tools.wmflabs.org/magnustools/oauth_uploader.php' , oauth_uploader_api : 'https://tools.wmflabs.org/magnustools/oauth_uploader.php?botmode=1&callback=?' , - flickr_api_url : 'https://secure.flickr.com/services/rest' , + flickr_api_url : 'https://flickr.com/services/rest' , flickr_api_key : '' , is_authorized : false , userinfo : {} , diff --git a/public_html/resources/vue/typeahead-search.html b/public_html/resources/vue/typeahead-search.html index d782204..1948793 100644 --- a/public_html/resources/vue/typeahead-search.html +++ b/public_html/resources/vue/typeahead-search.html @@ -69,7 +69,7 @@ Vue.component ( 'typeahead-search' , { template : '#typeahead-search-template' , - props : [ 'type' , 'fulltext' , 'initial' , 'language' , 'limit' , 'noinitialsearch' , 'initial_q' ] , + props : [ 'type' , 'fulltext' , 'initial' , 'language' , 'limit' , 'noinitialsearch' , 'initial_q' , 'nofocus' ] , data : function () { return { query:'' , last_query:'' , results:[] , q:'' , is_search_running:false , is_search_done:false , timeout:'' , key_select_position:-1 , last_emit:'' } } , created : function () { var me = this ; @@ -88,7 +88,7 @@ } } , mounted : function () { - this.$refs.search.focus(); + if ( typeof this.nofocus == 'undefined' ) this.$refs.search.focus(); if (typeof tt!='undefined') tt.updateInterface(this.$el) ; } , updated : function () { if (typeof tt!='undefined') tt.updateInterface(this.$el) } , diff --git a/public_html/unused_files.php b/public_html/unused_files.php index 60691ae..bc0317f 100644 --- a/public_html/unused_files.php +++ b/public_html/unused_files.php @@ -2,6 +2,7 @@ error_reporting(E_ERROR|E_CORE_ERROR|E_ALL|E_COMPILE_ERROR); ini_set('display_errors', 'On'); +ini_set('memory_limit','1500M'); include_once ( 'php/common.php' ) ; @@ -12,7 +13,8 @@ function get_image_row_div ( $file , $w , $h , $num ) { if ( $w < $h ) { $tw = round ( $thumbsize * $w / $h ) ; } - $thumb_url = get_thumbnail_url ( 'commons' , $file , $tw , 'wikimedia' ) ; + if ( preg_match('|\.ogg$|i',$file) ) $thumb_url = 'https://commons.wikimedia.org/w/resources/assets/file-type-icons/fileicon-ogg.png' ; + else $thumb_url = get_thumbnail_url ( 'commons' , $file , $tw , 'wikimedia' ) ; $h = "
" ; $h .= "
" ; @@ -22,9 +24,10 @@ function get_image_row_div ( $file , $w , $h , $num ) { $h .= "
" ; $h .= "$file" ; $fn = preg_replace('/\.[^.]+$/','',$file) ; - $h .= "
Search Wikipedias for this title to find potential articles to insert it." ; + if ( preg_match('|^[a-z]{2,3}-(.+?)\.ogg|i',$file,$m) ) $fn = $m[1] ; # Strip language code from ogg files + $h .= "
Search Wikipedias for this title to find potential articles to insert it." ; $fn = implode ( ' OR ' , explode ( ' ' , $fn ) ) ; - $h .= " (OR version)" ; + $h .= " (OR version)" ; $h .= "
" ; $h .= "
" ; return $h ; diff --git a/scripts/buggregator/Buggregator.php b/scripts/buggregator/Buggregator.php new file mode 100644 index 0000000..f65d5e9 --- /dev/null +++ b/scripts/buggregator/Buggregator.php @@ -0,0 +1,269 @@ +extract_times() ; + foreach ( $times as $time ) { + if ( $this->date_created==self::ZERO_TIME or $this->date_created>$time ) $this->date_created = $time ; + if ( $this->date_last<$time ) $this->date_last = $time ; + } + } + + protected function add_associated_urls ( $buggregator ) { + $urls = $this->get_associated_urls ( $buggregator ) ; + foreach ( $urls as $url ) { + $url = $buggregator->escape ( $url ) ; + $sql = "INSERT IGNORE INTO `url_issue` (`url`,`issue_id`) VALUES ('{$url}',{$this->issue_id})" ; + $buggregator->getSQL ( $sql ) ; + } + } + + protected function add_associated_users ( $buggregator ) { + $user_id2role = $this->get_associated_users ( $buggregator ) ; + foreach ( $user_id2role AS $user_id => $role ) { + $user_id *= 1 ; + $role = $buggregator->escape ( $role ) ; + $sql = "INSERT IGNORE INTO `user_issue` (`user_id`,`issue_id`,`role`) VALUES ({$user_id},{$this->issue_id},'{$role}')" ; + $buggregator->getSQL ( $sql ) ; + } + } + + protected function create_as_new_issue ( $buggregator ) { + $this->set_times() ; + $this->construct_url ( $buggregator ) ; + $this->determine_tool ( $buggregator ) ; + $values = [] ; + foreach ( self::FIELDS AS $field ) $values[] = $buggregator->escape($this->$field); + $fields = implode('`,`',self::FIELDS) ; + $values = implode("','",$values) ; + $sql = "INSERT IGNORE INTO `issue` (`{$fields}`) VALUES ('{$values}')" ; + $buggregator->getSQL ( $sql ) ; + $this->issue_id = $buggregator->last_insert_id() ; + $this->add_associated_urls ( $buggregator ) ; + $this->add_associated_users ( $buggregator ) ; + return $this->issue_id ; + } +} + +class WikiIssue extends issue { + protected $wikitext ; + protected $wiki_page_id ; + + public static function new ( $wiki_page_id , $label , $wikitext ) { + $ret = new WikiIssue ; + $ret->label = trim($label) ; + $ret->wikitext = trim($wikitext) ; + $ret->wiki_page_id = $wiki_page_id * 1 ; + return $ret ; + } + + public function get_or_create_issue_id ( $buggregator ) { + if ( isset($this->issue_id) ) return $this->issue_id ; # Already has an issue ID + + # Paranoia + if ( !isset($this->label) ) throw new Exception("{__METHOD__}: No label set"); + if ( !isset($this->wiki_page_id) ) throw new Exception("{__METHOD__}: No wiki_page_id set"); + + # Try page/label + $safe_label = $buggregator->escape($this->label) ; + $sql = "SELECT * FROM `issue`,`wiki_issue` WHERE `site`='WIKI' AND `label`='{$safe_label}' AND `issue_id`=`issue`.`id` AND `wiki_page_id`={$this->wiki_page_id}" ; + $result = $buggregator->getSQL ( $sql ) ; + if($o = $result->fetch_object()) { + $this->issue_id = $o->id ; + return $this->issue_id ; + } + + # Create new issue + $this->description = $buggregator->tfc->trimWikitextMarkup($this->wikitext); + $this->create_as_new_issue($buggregator) ; + + # Create new wiki issue + $wikitext = $buggregator->escape ( $this->wikitext ) ; + $sql = "INSERT IGNORE INTO `wiki_issue` (`wiki_page_id`,`issue_id`,`wikitext`) VALUES ({$this->wiki_page_id},{$this->issue_id},'{$wikitext}')" ; + $buggregator->getSQL ( $sql ) ; + } + + protected function construct_url ( $buggregator ) { + $this->url = '' ; + $sql = "SELECT * FROM `wiki_page` WHERE `id`={$this->wiki_page_id}" ; + $result = $buggregator->getSQL ( $sql ) ; + if($o = $result->fetch_object()){ + $server = $buggregator->tfc->getWebserverForWiki($o->wiki) ; + if ( $server == '' ) return ; + $page = urlencode($o->page) ; + $hash = urlencode($this->label) ; + $this->url = "https://{$server}/wiki/{$page}#{$hash}" ; + } + } + + protected function determine_tool ( $buggregator ) { + $this->tool = 0 ; + $sql = "SELECT * FROM `wiki_page` WHERE `id`={$this->wiki_page_id}" ; + $result = $buggregator->getSQL ( $sql ) ; + if($o = $result->fetch_object()) $this->tool = $o->tool_hint ; + } + + protected function get_associated_urls ( $buggregator ) { + if ( preg_match_all('#\bhttps?://[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/))#', $this->wikitext, $m) ) { + return $m[0] ; + } + return [] ; + } + + protected function get_associated_users ( $buggregator ) { + $user_id2role = [] ; + if ( preg_match_all('#\[\[(User|Benutzer):(.+?)(\||\]\])#',$this->wikitext,$m) ) { + $role = 'CREATOR' ; # Assuming first user name is the creator + $usernames = $m[2] ; + foreach ( $usernames AS $username ) { + $username = trim ( $username ) ; + if ( $username == '' ) continue ; + $user = new User ( $username , 'WIKI' ) ; + $user_id = $user->get_or_create_user_id ( $buggregator ) ; + if ( isset($user_id2role[$user_id]) ) continue ; + $user_id2role[$user_id] = $role ; + $role = 'UNKNOWN' ; + } + } + return $user_id2role ; + } + + protected function extract_times () { + $ret = [] ; + if ( preg_match_all('|(\d{2}:\d{2}, \d{1,2} \S+? \d{4}) \((UTC|CET|CEST)\)|',$this->wikitext,$m) ) { + foreach ( $m[1] AS $time ) { + $ret[] = date('Y-m-d H:i:00',strtotime($time)); + } + } + return $ret ; + } + +} + +class User { + protected $user_id ; + protected $username ; + protected $site ; + + public function __construct ( $username , $site ) { + $this->site = $site ; + $this->username = $this->sanitize($username) ; + } + + protected function sanitize ( $username ) { + if ( $this->site == 'WIKI' ) { + $username = str_replace ( '_' , ' ' , $username ) ; + } + $username = trim ( $username ) ; + return $username ; + } + + public function get_or_create_user_id ( $buggregator ) { + $username = $buggregator->escape($this->username) ; + $site = $buggregator->escape($this->site) ; + $sql = "SELECT * FROM `user` WHERE `name`='{$username}' AND `site`='{$site}'" ; + $result = $buggregator->getSQL ( $sql ) ; + if($o = $result->fetch_object()) { + $this->user_id = $o->id ; + } else { + $sql = "INSERT IGNORE INTO `user` (`name`,`site`) VALUES ('{$username}','{$site}')" ; + $buggregator->getSQL ( $sql ) ; + $this->user_id = $buggregator->last_insert_id() ; + } + return $this->user_id ; + } +} + +class Buggregator { + public $tfc ; + protected $tool_db ; + + public function __construct () { + $this->tfc = new ToolforgeCommon ( 'buggregator' ) ; + $this->tool_db = $this->tfc->openDBtool('buggregator') ; + } + + public function update_from_wikipages() { + $sql = "SELECT * FROM `wiki_page`" ; + $result = $this->getSQL ( $sql ) ; + while($o = $result->fetch_object()){ + $wikitext = $this->tfc->getWikiPageText($o->wiki,$o->page) ; + $rows = explode ( "\n" , $wikitext ) ; + $label = '' ; + $wikitext = '' ; + foreach ( $rows AS $row ) { + if ( preg_match('|^\s*={2,}\s*(.+?)\s*={2,}\s*$|',$row,$m) ) { + if ( $label != '' ) { + $issue = WikiIssue::new($o->id,$label,$wikitext) ; + $issue->get_or_create_issue_id($this); + } + $label = $m[1] ; + $wikitext = '' ; + continue ; + } + $wikitext .= "{$row}\n" ; + } + if ( $label != '' ) { + $issue = WikiIssue::new($o->id,$label,$wikitext) ; + $issue->get_or_create_issue_id($this); + } + } + } + + public function getSQL ( $sql ) { + return $this->tfc->getSQL ( $this->tool_db , $sql ) ; + } + + public function escape ( $s ) { + return $this->tool_db->real_escape_string ( $s ) ; + } + + public function last_insert_id () { + return $this->tool_db->insert_id ; + } + +} + +?> \ No newline at end of file diff --git a/scripts/buggregator/update.php b/scripts/buggregator/update.php new file mode 100755 index 0000000..6c7b02d --- /dev/null +++ b/scripts/buggregator/update.php @@ -0,0 +1,10 @@ +#!/usr/bin/env php +update_from_wikipages() ; + +?> \ No newline at end of file From 304b588019440d3faa2ba82d2823c4b9cff7a564 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Thu, 24 Jun 2021 10:11:01 +0000 Subject: [PATCH 02/45] misc buggregator --- scripts/buggregator/Buggregator.php | 156 ++++++++++++++++++++++++++-- scripts/buggregator/update.php | 3 +- 2 files changed, 150 insertions(+), 9 deletions(-) diff --git a/scripts/buggregator/Buggregator.php b/scripts/buggregator/Buggregator.php index f65d5e9..7855a3f 100644 --- a/scripts/buggregator/Buggregator.php +++ b/scripts/buggregator/Buggregator.php @@ -7,7 +7,7 @@ class Issue { const ZERO_TIME = '0000-00-00 00:00:00' ; - const FIELDS = ['label','status','date_created','date_last','site','url','description','tool'] ; + const FIELDS = ['label','status','date_created','date_last','site','url','description','tool','priority'] ; protected $issue_id ; protected $label ; @@ -18,6 +18,26 @@ class Issue { protected $url = '' ; protected $description = '' ; protected $tool = 0 ; + protected $priority = 'NORMAL' ; + + public function id() { return $this->issue_id * 1 ; } + public function label() { return $this->label ; } + public function status() { return $this->status ; } + public function date_created() { return $this->date_created ; } + public function date_last() { return $this->date_last ; } + public function site() { return $this->site ; } + public function url() { return $this->url ; } + public function description() { return $this->description ; } + public function tool() { return $this->tool * 1 ; } + public function priority() { return $this->priority ; } + + public function set_status ( $new_status ) { $this->status = trim(strtoupper($new_status)) ; } + + public static function new_from_object ( $o ) { + $ret = new self ; + $ret->fill_from_object($o) ; + return $ret ; + } public function get_or_create_issue_id ( $buggregator ) { throw new Exception("{__METHOD__} should never be called directly!") ; @@ -31,7 +51,7 @@ protected function construct_url ( $buggregator ) { throw new Exception("{__METHOD__} should never be called directly!") ; } - protected function determine_tool ( $buggregator ) { + public function determine_tool ( $buggregator ) { throw new Exception("{__METHOD__} should never be called directly!") ; } @@ -43,7 +63,16 @@ protected function get_associated_users ( $buggregator ) { throw new Exception("{__METHOD__} should never be called directly!") ; } - protected function set_times () { + protected function fill_from_object ( $o ) { + if ( !isset($o->id) ) throw new Exception("{__METHOD__}: `id` not in given object") ; + $this->issue_id = $o->id ; + foreach ( self::FIELDS AS $field ) { + if ( !isset($o->$field) ) throw new Exception("{__METHOD__}: Field {$field} not in given object") ; + $this->$field = $o->$field ; + } + } + + public function set_times () { $times = $this->extract_times() ; foreach ( $times as $time ) { if ( $this->date_created==self::ZERO_TIME or $this->date_created>$time ) $this->date_created = $time ; @@ -85,11 +114,26 @@ protected function create_as_new_issue ( $buggregator ) { $this->add_associated_users ( $buggregator ) ; return $this->issue_id ; } + + public function update_in_database ( $buggregator , $fields = self::FIELDS ) { + if ( $this->issue_id == 0 ) throw new Exception("{__METHOD__}: No issue ID"); + $values = [] ; + foreach ( $fields AS $field ) $values[] = "`{$field}`='".$buggregator->escape($this->$field)."'"; + $sql = "UPDATE `issue` SET " . implode(',',$values) . " WHERE `id`={$this->issue_id}" ; + $buggregator->getSQL ( $sql ) ; + } } class WikiIssue extends issue { protected $wikitext ; protected $wiki_page_id ; + const DE_MONTHS = [ 'Mär' => 'Mar','Mai' => 'May','Okt' => 'Oct','Dez' => 'Dec'] ; + + public static function new_from_object ( $o ) { + $ret = new self ; + $ret->fill_from_object($o) ; + return $ret ; + } public static function new ( $wiki_page_id , $label , $wikitext ) { $ret = new WikiIssue ; @@ -99,6 +143,9 @@ public static function new ( $wiki_page_id , $label , $wikitext ) { return $ret ; } + public function wikitext() { return $this->wikitext ; } + public function wiki_page_id() { return $this->wiki_page_id * 1 ; } + public function get_or_create_issue_id ( $buggregator ) { if ( isset($this->issue_id) ) return $this->issue_id ; # Already has an issue ID @@ -132,17 +179,31 @@ protected function construct_url ( $buggregator ) { if($o = $result->fetch_object()){ $server = $buggregator->tfc->getWebserverForWiki($o->wiki) ; if ( $server == '' ) return ; - $page = urlencode($o->page) ; - $hash = urlencode($this->label) ; + $page = $buggregator->tfc->urlEncode($o->page) ; + $hash = $buggregator->tfc->urlEncode($this->label) ; $this->url = "https://{$server}/wiki/{$page}#{$hash}" ; } } - protected function determine_tool ( $buggregator ) { + public function determine_tool ( $buggregator ) { + # Try wiki page hint $this->tool = 0 ; $sql = "SELECT * FROM `wiki_page` WHERE `id`={$this->wiki_page_id}" ; $result = $buggregator->getSQL ( $sql ) ; if($o = $result->fetch_object()) $this->tool = $o->tool_hint ; + + # Try tool name + $toolname2id = $buggregator->get_unique_tool_name_ids() ; + $candidate_tools = [] ; + foreach ( $toolname2id as $tool_name => $tool_id ) { + $pattern = "|\b{$tool_name}\b|i" ; + if ( !preg_match($pattern,$this->wikitext,$m) ) continue ; + $candidate_tools[] = $tool_id ; + } + $candidate_tools = array_values ( array_unique($candidate_tools) ) ; + if ( count($candidate_tools) == 1 ) { + $this->tool = $candidate_tools[0] * 1 ; + } } protected function get_associated_urls ( $buggregator ) { @@ -172,14 +233,24 @@ protected function get_associated_users ( $buggregator ) { protected function extract_times () { $ret = [] ; - if ( preg_match_all('|(\d{2}:\d{2}, \d{1,2} \S+? \d{4}) \((UTC|CET|CEST)\)|',$this->wikitext,$m) ) { + if ( preg_match_all('|(\d{2}:\d{2}, \d{1,2}\.* \S+ \d{4}) \([A-Z]+\)|',$this->wikitext,$m) ) { foreach ( $m[1] AS $time ) { - $ret[] = date('Y-m-d H:i:00',strtotime($time)); + foreach ( self::DE_MONTHS AS $from => $to ) $time = str_replace($from,$to,$time) ; + $time = strtotime($time) ; + $ret[] = Buggregator::format_time($time) ; } } return $ret ; } + protected function fill_from_object ( $o ) { + parent::fill_from_object ( $o ) ; + foreach ( ['wikitext','wiki_page_id'] AS $field ) { + if ( !isset($o->$field) ) throw new Exception("{__METHOD__}: Field {$field} not in given object") ; + $this->$field = $o->$field ; + } + } + } class User { @@ -219,12 +290,19 @@ public function get_or_create_user_id ( $buggregator ) { class Buggregator { public $tfc ; protected $tool_db ; + protected $toolname2id = [] ; + + const IGNORE_TOOL_NAMES = ['data'] ; public function __construct () { $this->tfc = new ToolforgeCommon ( 'buggregator' ) ; $this->tool_db = $this->tfc->openDBtool('buggregator') ; } + static public function format_time ( $time ) { + return date('Y-m-d H:i:s',$time); + } + public function update_from_wikipages() { $sql = "SELECT * FROM `wiki_page`" ; $result = $this->getSQL ( $sql ) ; @@ -251,7 +329,69 @@ public function update_from_wikipages() { } } } + + public function get_unique_tool_name_ids() { + if ( count($this->toolname2id) == 0 ) { + $sql = "SELECT group_concat(`id`) AS `id`,lower(regexp_replace(`name`,'_',' ')) AS `name` from `tool` group by lower(regexp_replace(`name`,'_',' ')) having count(*)=1" ; + $this->toolname2id = [] ; + $result = $this->getSQL ( $sql ) ; + while($o = $result->fetch_object()) $this->toolname2id[$o->name] = $o->id ; + foreach ( self::IGNORE_TOOL_NAMES AS $bad_name ) unset($this->toolname2id[$bad_name]) ; + } + return $this->toolname2id ; + } + + protected function maintenance_wiki_dates () { + $sql = "SELECT * FROM `vw_wiki_issue` WHERE `date_created`='".Issue::ZERO_TIME."' AND `status`='OPEN'" ; + $result = $this->getSQL ( $sql ) ; + while($o = $result->fetch_object()){ + $issue = WikiIssue::new_from_object ( $o ) ; + if ( $issue->date_created() != WikiIssue::ZERO_TIME ) continue ; # Paranoia + $issue->set_times() ; + if ( $issue->date_created() == WikiIssue::ZERO_TIME ) continue ; # None found + $issue->update_in_database ( $this , ['date_created','date_last'] ) ; + } + } + + protected function maintenance_wiki_tool_guess () { + $sql = "SELECT * FROM `vw_wiki_issue` WHERE `tool`=0 AND `status`='OPEN'" ; + $result = $this->getSQL ( $sql ) ; + while($o = $result->fetch_object()) { + $issue = WikiIssue::new_from_object ( $o ) ; + if ( $issue->tool() != 0 ) continue ; # Paranoia + $issue->determine_tool ( $this ) ; + if ( $issue->tool() == 0 ) continue ; # No avail + $issue->update_in_database ( $this , ['tool'] ) ; + } + } + + protected function maintenance_wiki_close_old_replied () { + # Get all open issues where I wrote something... + $time = strtotime("-1 month"); + $cutoff_time = self::format_time($time); + $sql = "SELECT `vw_wiki_issue`.* FROM `vw_wiki_issue`,`user_issue`" ; + $sql .= " WHERE `date_last`!='".Issue::ZERO_TIME."' AND `date_last`<'{$cutoff_time}' AND `status`='OPEN'" ; + $sql .= " AND `user_issue`.`issue_id`=`vw_wiki_issue`.`id` AND `user_issue`.`user_id`=14" ; + $result = $this->getSQL ( $sql ) ; + while($o = $result->fetch_object()) { + $issue = WikiIssue::new_from_object ( $o ) ; + if ( $issue->status() != 'OPEN' ) continue ; # Paranoia + $rows = explode ( "\n" , trim($issue->wikitext()) ) ; + $last_row = array_pop($rows); + # Make sure I wrote on the last row + if ( !preg_match('|\bMagnus[ _]Manske\b|',$last_row) ) continue ; + # Old issue, I wrote the last line => closed + $issue->set_status ( 'CLOSED' ) ; + $issue->update_in_database ( $this , ['status'] ) ; + } + } + public function maintenance () { + $this->maintenance_wiki_dates() ; + $this->maintenance_wiki_tool_guess() ; + $this->maintenance_wiki_close_old_replied() ; + } + public function getSQL ( $sql ) { return $this->tfc->getSQL ( $this->tool_db , $sql ) ; } diff --git a/scripts/buggregator/update.php b/scripts/buggregator/update.php index 6c7b02d..5c0f4f3 100755 --- a/scripts/buggregator/update.php +++ b/scripts/buggregator/update.php @@ -5,6 +5,7 @@ $buggregator = new Buggregator ; -$buggregator->update_from_wikipages() ; +#$buggregator->update_from_wikipages() ; +$buggregator->maintenance() ; ?> \ No newline at end of file From 883ab3f721bbe40c6f3bd9462fba6c4115adfe54 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Thu, 24 Jun 2021 12:39:52 +0000 Subject: [PATCH 03/45] github scraping --- scripts/buggregator/Buggregator.php | 97 ++++++++++++++++++++++++++++- scripts/buggregator/update.php | 3 +- 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/scripts/buggregator/Buggregator.php b/scripts/buggregator/Buggregator.php index 7855a3f..2a07886 100644 --- a/scripts/buggregator/Buggregator.php +++ b/scripts/buggregator/Buggregator.php @@ -44,6 +44,7 @@ public function get_or_create_issue_id ( $buggregator ) { } protected function extract_times () { + # NEEDS TO RETURN AN ARRAY! throw new Exception("{__METHOD__} should never be called directly!") ; } @@ -56,7 +57,11 @@ public function determine_tool ( $buggregator ) { } protected function get_associated_urls ( $buggregator ) { - throw new Exception("{__METHOD__} should never be called directly!") ; + # DEFAULT: Scrape description for URLs, change as required! + if ( preg_match_all('#\bhttps?://[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/))#', $this->description, $m) ) { + return $m[0] ; + } + return [] ; } protected function get_associated_users ( $buggregator ) { @@ -124,6 +129,69 @@ public function update_in_database ( $buggregator , $fields = self::FIELDS ) { } } +class GitIssue extends issue { + protected $git_id ; + protected $git_repo_id ; + protected $tmp_user_names = [] ; + + protected function extract_times () { return [] ; } + protected function construct_url ( $buggregator ) {} + public function determine_tool ( $buggregator ) {} + + protected function get_associated_users ( $buggregator ) { + $user_id2role = [] ; + foreach ( $this->tmp_user_names AS $username ) { + $user = new User ( $username , $this->site ) ; + $user_id = $user->get_or_create_user_id ( $buggregator ) ; + if ( count($user_id2role) == 0 ) $user_id2role[$user_id] = 'CREATOR' ; + else if ( !isset($user_id2role[$user_id]) ) $user_id2role[$user_id] = 'UNKNOWN' ; + } + return $user_id2role ; + } + + public function get_or_create_issue_id ( $buggregator ) { + if ( isset($this->issue_id) ) return $this->issue_id ; # Already has an issue ID + + # Paranoia + if ( !isset($this->git_id) ) throw new Exception("{__METHOD__}: No git_id set"); + if ( !isset($this->git_repo_id) ) throw new Exception("{__METHOD__}: No git_repo_id set"); + + # Check if exists + $sql = "SELECT * FROM vw_git_issue WHERE site='{$this->site}' AND git_id={$this->git_id} AND git_repo_id={$this->git_repo_id}" ; + $result = $buggregator->getSQL ( $sql ) ; + if($o = $result->fetch_object()) { + $this->issue_id = $o->id ; + return $this->issue_id ; + } + + # Create new issue + $this->create_as_new_issue($buggregator) ; + + # Create new wiki issue + $sql = "INSERT IGNORE INTO `git_issue` (`issue_id`,`git_id`,`git_repo_id`) VALUES ({$this->issue_id},{$this->git_id},{$this->git_repo_id})" ; + $buggregator->getSQL ( $sql ) ; + return $this->issue_id ; + } +} + +class GithubIssue extends GitIssue { + public static function new_from_json ( $git_repo , $j ) { + $ret = new self ; + $ret->label = $j->title ; + $ret->url = $j->html_url ; + $ret->status = strtoupper($j->state) ; + $ret->date_created = Buggregator::format_time(strtotime($j->created_at)) ; + $ret->date_last = Buggregator::format_time(strtotime($j->updated_at)) ; + $ret->description = $j->body ; + $ret->tool = $git_repo->tool_id ; + $ret->site = 'GITHUB' ; + $ret->git_repo_id = $git_repo->id * 1 ; + $ret->git_id = $j->number * 1 ; + $ret->tmp_user_names = [ $j->user->login ] ; + return $ret ; + } +} + class WikiIssue extends issue { protected $wikitext ; protected $wiki_page_id ; @@ -170,6 +238,7 @@ public function get_or_create_issue_id ( $buggregator ) { $wikitext = $buggregator->escape ( $this->wikitext ) ; $sql = "INSERT IGNORE INTO `wiki_issue` (`wiki_page_id`,`issue_id`,`wikitext`) VALUES ({$this->wiki_page_id},{$this->issue_id},'{$wikitext}')" ; $buggregator->getSQL ( $sql ) ; + return $this->issue_id ; } protected function construct_url ( $buggregator ) { @@ -303,7 +372,7 @@ static public function format_time ( $time ) { return date('Y-m-d H:i:s',$time); } - public function update_from_wikipages() { + protected function update_from_wikipages() { $sql = "SELECT * FROM `wiki_page`" ; $result = $this->getSQL ( $sql ) ; while($o = $result->fetch_object()){ @@ -341,6 +410,28 @@ public function get_unique_tool_name_ids() { return $this->toolname2id ; } + protected function update_from_github () { + $sql = "SELECT * FROM `git_repo` WHERE `repo_type`='GITHUB'" ; + $result = $this->getSQL ( $sql ) ; + while($o = $result->fetch_object()){ + $opts = [ "http" => ["method" => "GET","header" => "Accept: application/vnd.github.v3+json\r\n" ]]; + $context = stream_context_create($opts); + $json = file_get_contents($o->api_issues_url, false, $context); + $json = json_decode ( $json ) ; + foreach ( $json AS $git_issue ) { + $issue = GithubIssue::new_from_json ( $o , $git_issue ) ; + $issue->get_or_create_issue_id ( $this ) ; + } + } + } + + public function update () { + #$this->update_from_wikipages() ; + $this->update_from_github() ; + #$this->maintenance() ; + } + + protected function maintenance_wiki_dates () { $sql = "SELECT * FROM `vw_wiki_issue` WHERE `date_created`='".Issue::ZERO_TIME."' AND `status`='OPEN'" ; $result = $this->getSQL ( $sql ) ; @@ -385,7 +476,7 @@ protected function maintenance_wiki_close_old_replied () { $issue->update_in_database ( $this , ['status'] ) ; } } - + public function maintenance () { $this->maintenance_wiki_dates() ; $this->maintenance_wiki_tool_guess() ; diff --git a/scripts/buggregator/update.php b/scripts/buggregator/update.php index 5c0f4f3..5e00b6e 100755 --- a/scripts/buggregator/update.php +++ b/scripts/buggregator/update.php @@ -5,7 +5,6 @@ $buggregator = new Buggregator ; -#$buggregator->update_from_wikipages() ; -$buggregator->maintenance() ; +$buggregator->update() ; ?> \ No newline at end of file From e968a5f840364e2af48c73c9d97250aceb86222a Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Thu, 24 Jun 2021 13:06:17 +0000 Subject: [PATCH 04/45] bitbucket scraping --- scripts/buggregator/Buggregator.php | 38 ++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/scripts/buggregator/Buggregator.php b/scripts/buggregator/Buggregator.php index 2a07886..f0e7195 100644 --- a/scripts/buggregator/Buggregator.php +++ b/scripts/buggregator/Buggregator.php @@ -192,6 +192,24 @@ public static function new_from_json ( $git_repo , $j ) { } } +class BitbucketIssue extends GitIssue { + public static function new_from_json ( $git_repo , $j ) { + $ret = new self ; + $ret->label = "{$j->title} [{$j->kind}/{$j->priority}]" ; + $ret->url = $j->links->html->href ; + $ret->status = 'OPEN' ; # default + $ret->date_created = Buggregator::format_time(strtotime($j->created_on)) ; + $ret->date_last = Buggregator::format_time(strtotime($j->updated_on)) ; + $ret->description = $j->content->raw ; + $ret->tool = $git_repo->tool_id ; + $ret->site = 'BITBUCKET' ; + $ret->git_repo_id = $git_repo->id * 1 ; + $ret->git_id = $j->id * 1 ; + # TODO creating user + return $ret ; + } +} + class WikiIssue extends issue { protected $wikitext ; protected $wiki_page_id ; @@ -425,9 +443,27 @@ protected function update_from_github () { } } + protected function update_from_bitbucket () { + $sql = "SELECT * FROM `git_repo` WHERE `repo_type`='BITBUCKET'" ; + $result = $this->getSQL ( $sql ) ; + while($o = $result->fetch_object()){ + $url = "{$o->api_issues_url}/?pagelen=100" ; + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); + $json = curl_exec($ch); + $json = json_decode ( $json ) ; + foreach ( $json->values AS $git_issue ) { + $issue = BitbucketIssue::new_from_json ( $o , $git_issue ) ; + $issue->get_or_create_issue_id ( $this ) ; + } + } + } + public function update () { #$this->update_from_wikipages() ; - $this->update_from_github() ; + #$this->update_from_github() ; + $this->update_from_bitbucket() ; #$this->maintenance() ; } From 8b8810a85de2bdb6db19866ec1e5e966f379fd1e Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Thu, 24 Jun 2021 13:27:33 +0000 Subject: [PATCH 05/45] update complete --- scripts/buggregator/Buggregator.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/buggregator/Buggregator.php b/scripts/buggregator/Buggregator.php index f0e7195..c46f1b0 100644 --- a/scripts/buggregator/Buggregator.php +++ b/scripts/buggregator/Buggregator.php @@ -453,6 +453,7 @@ protected function update_from_bitbucket () { curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); $json = curl_exec($ch); $json = json_decode ( $json ) ; + if ( !isset($json->values) ) continue ; foreach ( $json->values AS $git_issue ) { $issue = BitbucketIssue::new_from_json ( $o , $git_issue ) ; $issue->get_or_create_issue_id ( $this ) ; @@ -461,10 +462,10 @@ protected function update_from_bitbucket () { } public function update () { - #$this->update_from_wikipages() ; - #$this->update_from_github() ; + $this->update_from_wikipages() ; + $this->update_from_github() ; $this->update_from_bitbucket() ; - #$this->maintenance() ; + $this->maintenance() ; } From b664e76d7e4ac10b0280ffbdb6fe80ba35845181 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Thu, 24 Jun 2021 15:18:08 +0000 Subject: [PATCH 06/45] wikidata, partially --- scripts/buggregator/Buggregator.php | 60 +++++++++++++++++++++++++++-- scripts/buggregator/update.php | 3 +- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/scripts/buggregator/Buggregator.php b/scripts/buggregator/Buggregator.php index c46f1b0..71f5948 100644 --- a/scripts/buggregator/Buggregator.php +++ b/scripts/buggregator/Buggregator.php @@ -460,12 +460,64 @@ protected function update_from_bitbucket () { } } } + + protected function update_from_wikidata() { + $vtl = 'view-topiclist' ; + $top = 'topic-of-post' ; + $limit = 10 ; # TODO 100 + $url = "https://www.wikidata.org/w/api.php?action=flow&submodule=view-topiclist&page=User%20Talk:Magnus%20Manske&vtllimit={$limit}&format=json" ; + $json = json_decode ( file_get_contents ( $url ) ) ; + $json = $json->flow->$vtl->result->topiclist ; + $topics = [] ; + foreach ( $json->revisions AS $rev ) { + $topic_id = $rev->workflowId ; + $time = self::format_time(strtotime($rev->timestamp)) ; + $description = '' ; + $urls = [] ; + if ( $rev->content->format == 'fixed-html' ) { + $html = html_entity_decode($rev->content->content); + $description = trim(html_entity_decode(strip_tags($rev->content->content))) ; + if ( preg_match_all('#\bhttps?://[^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/))#', $html, $m) ) { + foreach ( $m[0] AS $url ) { + $url = preg_replace ( '|".*$|' , '' , $url ) ; + $urls[] = $url ; + } + } + } + if ( !isset($topics[$topic_id]) ) { + $topics[$topic_id] = [ + 'label' => $rev->properties->$top->plaintext , + 'date_created' => $time , + 'date_last' => $time , + 'site' => 'WIKIDATA' , + 'url' => "https://www.wikidata.org/wiki/Topic:{$topic_id}" , + 'status' => ($rev->isLocked?'CLOSED':'OPEN') , + 'description' => $description , + 'tmp_authors' => [ $rev->author->name ] , + 'tmp_urls' => $urls + ] ; + } else { + if ( $time > $topics[$topic_id]['date_last'] ) $topics[$topic_id]['date_last'] = $time ; + $topics[$topic_id]['tmp_authors'][] = $rev->author->name ; + $topics[$topic_id]['description'] .= "\n\n{$description}" ; + $topics[$topic_id]['tmp_urls'] = array_merge($topics[$topic_id]['tmp_urls'],$urls) ; + } + } + # Cleanup + foreach ( $topics AS $topic_id => $topic ) { + $topics[$topic_id]['tmp_authors'] = array_unique ( $topic['tmp_authors'] ) ; + $topics[$topic_id]['tmp_urls'] = array_unique ( $topic['tmp_urls'] ) ; + $topics[$topic_id]['description'] = trim($topic['description']) ; + } + print_r ( $topics ) ; + } public function update () { - $this->update_from_wikipages() ; - $this->update_from_github() ; - $this->update_from_bitbucket() ; - $this->maintenance() ; + $this->update_from_wikidata() ; + #$this->update_from_wikipages() ; + #$this->update_from_github() ; + #$this->update_from_bitbucket() ; + #$this->maintenance() ; } diff --git a/scripts/buggregator/update.php b/scripts/buggregator/update.php index 5e00b6e..dc7ede0 100755 --- a/scripts/buggregator/update.php +++ b/scripts/buggregator/update.php @@ -5,6 +5,7 @@ $buggregator = new Buggregator ; -$buggregator->update() ; +if ( $argv[1] == 'update' ) $buggregator->update() ; +else print "Nothing to do!\n" ; ?> \ No newline at end of file From de7ed3225fbab0a45f6f029d4a95c4e7194e4ee4 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Thu, 24 Jun 2021 17:52:24 +0000 Subject: [PATCH 07/45] misc --- scripts/buggregator/Buggregator.php | 94 ++++++++++++++++++++++++++--- 1 file changed, 84 insertions(+), 10 deletions(-) diff --git a/scripts/buggregator/Buggregator.php b/scripts/buggregator/Buggregator.php index 71f5948..33fff05 100644 --- a/scripts/buggregator/Buggregator.php +++ b/scripts/buggregator/Buggregator.php @@ -129,7 +129,57 @@ public function update_in_database ( $buggregator , $fields = self::FIELDS ) { } } -class GitIssue extends issue { +class WikidataIssue extends Issue { + protected $tmp_authors = [] ; + protected $tmp_urls = [] ; + + public static function new_from_topic ( $topic ) { + $ret = new self ; + foreach ( $topic AS $k => $v ) { + $ret->$k = $v ; + } + return $ret ; + } + + protected function extract_times () { return [] ; } + protected function construct_url ( $buggregator ) {} + public function determine_tool ( $buggregator ) {} + + protected function get_associated_urls ( $buggregator ) { + return $this->tmp_urls ; + } + + protected function get_associated_users ( $buggregator ) { + $user_id2role = [] ; + foreach ( $this->tmp_authors AS $username ) { + $user = new User ( $username , 'WIKI' ) ; + $user_id = $user->get_or_create_user_id ( $buggregator ) ; + $user_id2role[$user_id] = 'UNKNOWN' ; + } + return $user_id2role ; + } + + public function get_or_create_issue_id ( $buggregator ) { + if ( isset($this->issue_id) ) return $this->issue_id ; # Already has an issue ID + + # Paranoia + if ( !isset($this->url) ) throw new Exception("{__METHOD__}: No url set"); + + # Check if exists + $url = $buggregator->escape ( $this->url ) ; + $sql = "SELECT * FROM `issue` WHERE `site`='{$this->site}' AND `url`='{$url}'" ; + $result = $buggregator->getSQL ( $sql ) ; + if($o = $result->fetch_object()) { + $this->issue_id = $o->id ; + return $this->issue_id ; + } + + # Create new issue + return $this->create_as_new_issue($buggregator) ; + } +} + +class GitIssue extends Issue { protected $git_id ; protected $git_repo_id ; protected $tmp_user_names = [] ; @@ -210,7 +260,7 @@ public static function new_from_json ( $git_repo , $j ) { } } -class WikiIssue extends issue { +class WikiIssue extends Issue { protected $wikitext ; protected $wiki_page_id ; const DE_MONTHS = [ 'Mär' => 'Mar','Mai' => 'May','Okt' => 'Oct','Dez' => 'Dec'] ; @@ -464,9 +514,14 @@ protected function update_from_bitbucket () { protected function update_from_wikidata() { $vtl = 'view-topiclist' ; $top = 'topic-of-post' ; - $limit = 10 ; # TODO 100 + $limit = 100 ; $url = "https://www.wikidata.org/w/api.php?action=flow&submodule=view-topiclist&page=User%20Talk:Magnus%20Manske&vtllimit={$limit}&format=json" ; - $json = json_decode ( file_get_contents ( $url ) ) ; + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); + $json = curl_exec($ch); + $json = json_decode ( $json ) ; + if ( !isset($json) or !isset($json->flow) ) throw new Exception("{__METHOD__} failed to get {$url}") ; $json = $json->flow->$vtl->result->topiclist ; $topics = [] ; foreach ( $json->revisions AS $rev ) { @@ -509,15 +564,34 @@ protected function update_from_wikidata() { $topics[$topic_id]['tmp_urls'] = array_unique ( $topic['tmp_urls'] ) ; $topics[$topic_id]['description'] = trim($topic['description']) ; } - print_r ( $topics ) ; + foreach ( $topics AS $topic ) { + $issue = WikidataIssue::new_from_topic($topic) ; + $issue->get_or_create_issue_id($this); + } } public function update () { - $this->update_from_wikidata() ; - #$this->update_from_wikipages() ; - #$this->update_from_github() ; - #$this->update_from_bitbucket() ; - #$this->maintenance() ; + try { + $this->update_from_wikipages() ; + } catch (Exception $e) { + print $e->getMessage() . "\n" ; + } + try { + $this->update_from_github() ; + } catch (Exception $e) { + print $e->getMessage() . "\n" ; + } + try { + $this->update_from_bitbucket() ; + } catch (Exception $e) { + print $e->getMessage() . "\n" ; + } + try { + $this->update_from_wikidata() ; + } catch (Exception $e) { + print $e->getMessage() . "\n" ; + } + $this->maintenance() ; } From 6f740f97b8be2ceec0473d23d51a249c407c8150 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Fri, 25 Jun 2021 08:40:06 +0000 Subject: [PATCH 08/45] guessing tool from description --- scripts/buggregator/Buggregator.php | 48 ++++++++++++++++++++++------- scripts/buggregator/update.php | 1 + 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/scripts/buggregator/Buggregator.php b/scripts/buggregator/Buggregator.php index 33fff05..ed83faf 100644 --- a/scripts/buggregator/Buggregator.php +++ b/scripts/buggregator/Buggregator.php @@ -104,6 +104,22 @@ protected function add_associated_users ( $buggregator ) { } } + static public function determine_tool_from_text ( $text , $buggregator ) { + $ret = 0 ; + $toolname2id = $buggregator->get_unique_tool_name_ids() ; + $candidate_tools = [] ; + foreach ( $toolname2id as $tool_name => $tool_id ) { + $pattern = "|\b{$tool_name}\b|i" ; + if ( !preg_match($pattern,$text,$m) ) continue ; + $candidate_tools[] = $tool_id ; + } + $candidate_tools = array_values ( array_unique($candidate_tools) ) ; + if ( count($candidate_tools) == 1 ) { + $ret = $candidate_tools[0] * 1 ; + } + return $ret ; + } + protected function create_as_new_issue ( $buggregator ) { $this->set_times() ; $this->construct_url ( $buggregator ) ; @@ -330,17 +346,7 @@ public function determine_tool ( $buggregator ) { if($o = $result->fetch_object()) $this->tool = $o->tool_hint ; # Try tool name - $toolname2id = $buggregator->get_unique_tool_name_ids() ; - $candidate_tools = [] ; - foreach ( $toolname2id as $tool_name => $tool_id ) { - $pattern = "|\b{$tool_name}\b|i" ; - if ( !preg_match($pattern,$this->wikitext,$m) ) continue ; - $candidate_tools[] = $tool_id ; - } - $candidate_tools = array_values ( array_unique($candidate_tools) ) ; - if ( count($candidate_tools) == 1 ) { - $this->tool = $candidate_tools[0] * 1 ; - } + $this->tool = self::determine_tool_from_text ( $this->wikitext , $buggregator ) ; } protected function get_associated_urls ( $buggregator ) { @@ -619,6 +625,25 @@ protected function maintenance_wiki_tool_guess () { } } + protected function maintenance_not_wiki_tool_guess () { + # WIKI gets special treatment, see above + $sql = "SELECT `id`,`description` FROM `issue` WHERE `tool`=0 AND `status`='OPEN' AND `site`!='WIKI'" ; + $result = $this->getSQL ( $sql ) ; + while($o = $result->fetch_object()) { + $tool_id = Issue::determine_tool_from_text ( $o->description , $this ) ; + if ( $tool_id == 0 ) continue ; # No avail + $this->setIssueValue ( $o->id , 'tool' , $tool_id ) ; + } + } + + protected function setIssueValue ( $issue_id , $field , $value ) { + if ( !in_array($field, Issue::FIELDS) ) throw new Exception("{__METHOD__}: '{$field}'' is not a valid field in Issue") ; + $issue_id *= 1 ; + $value = $this->escape ( $value ) ; + $sql = "UPDATE `issue` SET `{$field}`='{$value}' WHERE `id`={$issue_id}" ; + $this->getSQL ( $sql ) ; + } + protected function maintenance_wiki_close_old_replied () { # Get all open issues where I wrote something... $time = strtotime("-1 month"); @@ -643,6 +668,7 @@ protected function maintenance_wiki_close_old_replied () { public function maintenance () { $this->maintenance_wiki_dates() ; $this->maintenance_wiki_tool_guess() ; + $this->maintenance_not_wiki_tool_guess() ; $this->maintenance_wiki_close_old_replied() ; } diff --git a/scripts/buggregator/update.php b/scripts/buggregator/update.php index dc7ede0..91493f4 100755 --- a/scripts/buggregator/update.php +++ b/scripts/buggregator/update.php @@ -6,6 +6,7 @@ $buggregator = new Buggregator ; if ( $argv[1] == 'update' ) $buggregator->update() ; +else if ( $argv[1] == 'maintenance' ) $buggregator->maintenance() ; else print "Nothing to do!\n" ; ?> \ No newline at end of file From cc83f99e2fb67afe961ec40e0952af28d7cd93df Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Fri, 25 Jun 2021 08:44:49 +0000 Subject: [PATCH 09/45] guessing tool from label --- scripts/buggregator/Buggregator.php | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/buggregator/Buggregator.php b/scripts/buggregator/Buggregator.php index ed83faf..79acc72 100644 --- a/scripts/buggregator/Buggregator.php +++ b/scripts/buggregator/Buggregator.php @@ -345,7 +345,11 @@ public function determine_tool ( $buggregator ) { $result = $buggregator->getSQL ( $sql ) ; if($o = $result->fetch_object()) $this->tool = $o->tool_hint ; - # Try tool name + # Try issue label + $this->tool = Issue::determine_tool_from_text ( $this->label , $buggregator ) ; + if ( $this->tool != 0 ) return ; + + # Try issue description $this->tool = self::determine_tool_from_text ( $this->wikitext , $buggregator ) ; } @@ -627,10 +631,11 @@ protected function maintenance_wiki_tool_guess () { protected function maintenance_not_wiki_tool_guess () { # WIKI gets special treatment, see above - $sql = "SELECT `id`,`description` FROM `issue` WHERE `tool`=0 AND `status`='OPEN' AND `site`!='WIKI'" ; + $sql = "SELECT `id`,`label`,`description` FROM `issue` WHERE `tool`=0 AND `status`='OPEN' AND `site`!='WIKI'" ; $result = $this->getSQL ( $sql ) ; while($o = $result->fetch_object()) { - $tool_id = Issue::determine_tool_from_text ( $o->description , $this ) ; + $tool_id = Issue::determine_tool_from_text ( $o->label , $this ) ; + if ( $tool_id == 0 ) $tool_id = Issue::determine_tool_from_text ( $o->description , $this ) ; if ( $tool_id == 0 ) continue ; # No avail $this->setIssueValue ( $o->id , 'tool' , $tool_id ) ; } From 23312b86198f32ab8819f59a00ee3e9338e9ae08 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Fri, 25 Jun 2021 12:31:01 +0000 Subject: [PATCH 10/45] MVP web interface buggregator --- public_html/buggregator/api.php | 55 +++++++++++++++++++++++++++++ public_html/buggregator/index.html | 39 ++++++++++++++++++++ public_html/buggregator/vue.js | 30 ++++++++++++++++ scripts/buggregator/Buggregator.php | 8 +++-- 4 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 public_html/buggregator/api.php create mode 100644 public_html/buggregator/index.html create mode 100644 public_html/buggregator/vue.js diff --git a/public_html/buggregator/api.php b/public_html/buggregator/api.php new file mode 100644 index 0000000..b178e1d --- /dev/null +++ b/public_html/buggregator/api.php @@ -0,0 +1,55 @@ +tfc->getRequest('action','') ; +$out = [ 'status' => 'OK' , 'data' => [] ] ; + +if ( $action == 'get_issues' ) { + $limit = $buggregator->tfc->getRequest('limit',25)*1 ; + $offset = $buggregator->tfc->getRequest('offset',0)*1 ; + $sort_by = $buggregator->escape($buggregator->tfc->getRequest('sort_by','acsending')) ; + $sort_order = $buggregator->escape($buggregator->tfc->getRequest('sort_order','acsending')) ; + + $sql_core = [] ; + # TODO + # label + # status + # site + # description + # tool + # priority + if ( count($sql_core) == 0 ) $sql_core = '' ; + else $sql_core = ' WHERE (' . implode ( '),(' , $sql_core ) . ')' ; + + $sql = "SELECT * FROM `issue` {$sql_core}" ; + if ( $sort_by != '' ) { + $sql .= " ORDER BY `{$sort_by}`" ; + if ( $sort_order == 'descending' ) $sql .= " DESC" ; + } + $sql .= " LIMIT {$limit} OFFSET {$offset}" ; + $out['sql'] = $sql ; + $result = $buggregator->getSQL ( $sql ) ; + while($o = $result->fetch_object()) $out['data']['results'][] = $o ; + + $sql = "SELECT count(*) AS `cnt` FROM `issue` {$sql_core}" ; + $result = $buggregator->getSQL ( $sql ) ; + if($o = $result->fetch_object()) $out['data']['stats']['this_query'] = $o->cnt ; + + $sql = "SELECT count(*) AS `cnt` FROM `issue` WHERE `status`='OPEN'" ; + $result = $buggregator->getSQL ( $sql ) ; + if($o = $result->fetch_object()) $out['data']['stats']['total_open'] = $o->cnt ; + +} else { + $o->status = "Unknown action '{$action}'" ; +} + +header('Content-Type: application/json'); +echo json_encode($out) ; + +?> \ No newline at end of file diff --git a/public_html/buggregator/index.html b/public_html/buggregator/index.html new file mode 100644 index 0000000..dff9ba2 --- /dev/null +++ b/public_html/buggregator/index.html @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + \ No newline at end of file diff --git a/public_html/buggregator/vue.js b/public_html/buggregator/vue.js new file mode 100644 index 0000000..dd86d4b --- /dev/null +++ b/public_html/buggregator/vue.js @@ -0,0 +1,30 @@ +'use strict'; + +let router ; +let app ; +let wd = new WikiData() ; +let config = { + status:['OPEN','CLOSED'], + site:['WIKI','WIKIDATA','GITHUB','BITBUCKET'], + priority:['HIGH','NORMAL','LOW'], +} ; + +$(document).ready ( function () { + vue_components.toolname = 'buggregator' ; + Promise.all ( [ + vue_components.loadComponents ( ['wd-link','tool-translate','tool-navbar','typeahead-search', + 'vue_components/issue-list.html', + ] ) , + ] ) .then ( () => { + wd_link_wd = wd ; + const routes = [ + { path: '/', component: IssueList , props:true }, + ] ; + router = new VueRouter({routes}) ; + app = new Vue ( { router } ) .$mount('#app') ; + } ) ; + + // Logging + //$.getJSON ( 'https://tools.wmflabs.org/magnustools/logger.php?tool=picturesque&method=loaded&callback=?' , function(j){} ) ; + +} ) ; diff --git a/scripts/buggregator/Buggregator.php b/scripts/buggregator/Buggregator.php index 79acc72..416e019 100644 --- a/scripts/buggregator/Buggregator.php +++ b/scripts/buggregator/Buggregator.php @@ -479,11 +479,15 @@ protected function update_from_wikipages() { public function get_unique_tool_name_ids() { if ( count($this->toolname2id) == 0 ) { + $bad_names = self::IGNORE_TOOL_NAMES ; $sql = "SELECT group_concat(`id`) AS `id`,lower(regexp_replace(`name`,'_',' ')) AS `name` from `tool` group by lower(regexp_replace(`name`,'_',' ')) having count(*)=1" ; $this->toolname2id = [] ; $result = $this->getSQL ( $sql ) ; - while($o = $result->fetch_object()) $this->toolname2id[$o->name] = $o->id ; - foreach ( self::IGNORE_TOOL_NAMES AS $bad_name ) unset($this->toolname2id[$bad_name]) ; + while($o = $result->fetch_object()) { + if ( isset($this->toolname2id[$o->name]) ) $bad_names[] = $o->name ; # Name collision + $this->toolname2id[$o->name] = $o->id ; + } + foreach ( $bad_names AS $bad_name ) unset($this->toolname2id[$bad_name]) ; } return $this->toolname2id ; } From 65d41374b629861ccbe9a121b0a3bf9ba9eeb8ba Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Fri, 25 Jun 2021 12:31:18 +0000 Subject: [PATCH 11/45] MVP web interface buggregator --- .../vue_components/issue-list.html | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 public_html/buggregator/vue_components/issue-list.html diff --git a/public_html/buggregator/vue_components/issue-list.html b/public_html/buggregator/vue_components/issue-list.html new file mode 100644 index 0000000..bc2b3bd --- /dev/null +++ b/public_html/buggregator/vue_components/issue-list.html @@ -0,0 +1,182 @@ + + + From 81cb1e380a49e3dcd1aae8e4fcaaf8edb1d3f062 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Fri, 25 Jun 2021 15:22:18 +0000 Subject: [PATCH 12/45] buggregator interface functional, viewing only --- public_html/buggregator/api.php | 37 +++++++++++++++---- public_html/buggregator/vue.js | 12 +++--- .../vue_components/issue-list.html | 30 ++++++++------- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/public_html/buggregator/api.php b/public_html/buggregator/api.php index b178e1d..6390235 100644 --- a/public_html/buggregator/api.php +++ b/public_html/buggregator/api.php @@ -15,17 +15,21 @@ $offset = $buggregator->tfc->getRequest('offset',0)*1 ; $sort_by = $buggregator->escape($buggregator->tfc->getRequest('sort_by','acsending')) ; $sort_order = $buggregator->escape($buggregator->tfc->getRequest('sort_order','acsending')) ; + $tool = $buggregator->escape($buggregator->tfc->getRequest('tool','')) ; + $status = explode(',',$buggregator->escape($buggregator->tfc->getRequest('status',''))) ; + $site = explode(',',$buggregator->escape($buggregator->tfc->getRequest('site',''))) ; + $priority = explode(',',$buggregator->escape($buggregator->tfc->getRequest('priority',''))) ; $sql_core = [] ; - # TODO - # label - # status - # site - # description - # tool - # priority + # TODO: + # search + if ( $tool != '' ) $sql_core[] = "`tool`=" . ($tool*1) ; + if ( $status!=[''] ) $sql_core[] = "`status` IN ('".implode("','",$status)."')" ; + if ( $site!=[''] ) $sql_core[] = "`site` IN ('".implode("','",$site)."')" ; + if ( $priority!=[''] ) $sql_core[] = "`priority` IN ('".implode("','",$priority)."')" ; + if ( count($sql_core) == 0 ) $sql_core = '' ; - else $sql_core = ' WHERE (' . implode ( '),(' , $sql_core ) . ')' ; + else $sql_core = ' WHERE (' . implode ( ') AND (' , $sql_core ) . ')' ; $sql = "SELECT * FROM `issue` {$sql_core}" ; if ( $sort_by != '' ) { @@ -45,6 +49,23 @@ $result = $buggregator->getSQL ( $sql ) ; if($o = $result->fetch_object()) $out['data']['stats']['total_open'] = $o->cnt ; +} else if ( $action == 'get_config' ) { + $out['data'] = [ + 'status' => ['OPEN','CLOSED'] , + 'site' => ['WIKI','WIKIDATA','GITHUB','BITBUCKET'] , + 'priority' => ['HIGH','NORMAL','LOW'] , + 'tools' => [] + ] ; + $sql = "SELECT * FROM `tool` ORDER BY `name`,`subdomain`" ; + $result = $buggregator->getSQL ( $sql ) ; + while($o = $result->fetch_object()) { + $nice_name = "{$o->subdomain}/{$o->name}" ; + if ( strtolower($o->subdomain) == strtolower($o->name) ) $nice_name = $o->name ; + if ( $o->subdomain == '' ) $nice_name = $o->name ; + $o->nice_name = $nice_name ; + $out['data']['tools']["{$o->id}"] = $o ; + } + } else { $o->status = "Unknown action '{$action}'" ; } diff --git a/public_html/buggregator/vue.js b/public_html/buggregator/vue.js index dd86d4b..8410eb3 100644 --- a/public_html/buggregator/vue.js +++ b/public_html/buggregator/vue.js @@ -3,11 +3,7 @@ let router ; let app ; let wd = new WikiData() ; -let config = { - status:['OPEN','CLOSED'], - site:['WIKI','WIKIDATA','GITHUB','BITBUCKET'], - priority:['HIGH','NORMAL','LOW'], -} ; +let config = {} ; $(document).ready ( function () { vue_components.toolname = 'buggregator' ; @@ -15,6 +11,12 @@ $(document).ready ( function () { vue_components.loadComponents ( ['wd-link','tool-translate','tool-navbar','typeahead-search', 'vue_components/issue-list.html', ] ) , + new Promise(function(resolve, reject) { + $.get ( './api.php' , {action:'get_config'} , function(d) { + config = d.data ; + resolve(); + } ) ; + } ) ] ) .then ( () => { wd_link_wd = wd ; const routes = [ diff --git a/public_html/buggregator/vue_components/issue-list.html b/public_html/buggregator/vue_components/issue-list.html index bc2b3bd..4408dc0 100644 --- a/public_html/buggregator/vue_components/issue-list.html +++ b/public_html/buggregator/vue_components/issue-list.html @@ -16,29 +16,33 @@

+
+ +
-
-
- -
@@ -47,8 +51,8 @@

-
-
+
+
{{stats.this_query}} @@ -80,7 +84,7 @@

{{r.label}} - + {{r.status}}
{{r.priority}} @@ -89,16 +93,16 @@

{{r.date_last}} - 
+					

 						{{r.description}}
-
+
??? - TOOL {{r.tool}} + {{config.tools[r.tool].name}} @@ -113,7 +117,7 @@

let IssueList = Vue.extend ( { props : [] , - data : function () { return { filters : { label:'' , status:{} , sort_by:'date_last' , sort_order:'descending' , site:{} , description:'' , tool:'' , priority:{} , limit:25 , offset:0 } , + data : function () { return { filters : { search:'' , status:{} , sort_by:'date_last' , sort_order:'descending' , site:{} , tool:'' , priority:{} , limit:25 , offset:0 } , results:[] , loading:false , stats:{} , sort_by_options:['label','status','date_created','date_last','site','tool','priority'] , button_groups:['site','status','priority'] , From 6da4d33b3f9624e30d8d6c59e19059cc909a7958 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Mon, 28 Jun 2021 16:21:14 +0000 Subject: [PATCH 13/45] misc --- public_html/buggregator/api.php | 72 ++++++++++++++++--- public_html/buggregator/vue.js | 4 +- .../vue_components/issue-list.html | 35 +++++++-- .../vue_components/tool_picker.html | 0 scripts/buggregator/Buggregator.php | 41 +++++++++++ 5 files changed, 137 insertions(+), 15 deletions(-) create mode 100644 public_html/buggregator/vue_components/tool_picker.html diff --git a/public_html/buggregator/api.php b/public_html/buggregator/api.php index 6390235..a13fd11 100644 --- a/public_html/buggregator/api.php +++ b/public_html/buggregator/api.php @@ -4,15 +4,57 @@ ini_set('display_errors', 'On'); require_once ( '/data/project/magnustools/scripts/buggregator/Buggregator.php' ) ; +require_once ( '/data/project/magnustools/public_html/php/oauth.php' ) ; + +function user_can_edit () { + global $config , $oa , $out ; + $userinfo = $oa->getConsumerRights()->query->userinfo ; + if ( !isset($userinfo) ) return false ; + if ( !isset($userinfo->name) ) return false ; + if ( !in_array($userinfo->name,$config['write_access']) ) return false ; + $out['user'] = $userinfo->name ; + return true ; +} + +$oa = new MW_OAuth ( [ + 'tool'=>'buggregator', + 'language' => 'www' , + 'project' => 'mediawiki' , + 'apiUrl' => 'https://www.mediawiki.org/w/api.php' , + 'ini_file' => '/data/project/magnustools/buggregator_oauth.ini' +] ) ; + +if ( isset($_REQUEST['oauth_verifier']) and isset($_REQUEST['oauth_token']) ) { + $url = "https://magnustools.toolforge.org/buggregator/" ; + header( "Location: $url" ); + echo 'Please see ' . htmlspecialchars( $url ) . ''; + exit(0); +} $buggregator = new Buggregator ; $action = $buggregator->tfc->getRequest('action','') ; $out = [ 'status' => 'OK' , 'data' => [] ] ; +$config = [ + 'status' => ['OPEN','CLOSED'] , + 'site' => ['WIKI','WIKIDATA','GITHUB','BITBUCKET'] , + 'priority' => ['HIGH','NORMAL','LOW'] , + 'write_access' => ['Magnus Manske'] , + 'tools' => [] +] ; -if ( $action == 'get_issues' ) { - $limit = $buggregator->tfc->getRequest('limit',25)*1 ; - $offset = $buggregator->tfc->getRequest('offset',0)*1 ; +if ( $action == 'authorize' ) { + $oa->doAuthorizationRedirect('https://magnustools.toolforge.org/buggregator/api.php'); + exit(0); +#} else if ( $action == 'is_logged_in' ) { +# $out['data']['is_logged_in'] = $oa->isAuthOK() ; +# $out['data']['message'] = $oa->error ?? '' ; +# $out['data']['userinfo'] = $oa->userinfo ?? ( (object) [] ) ; +} else if ( $action == 'get_rights' ) { + $out['result'] = $oa->getConsumerRights() ; +} else if ( $action == 'get_issues' ) { + $limit = (int) $buggregator->tfc->getRequest('limit',25) ; + $offset = (int) $buggregator->tfc->getRequest('offset',0) ; $sort_by = $buggregator->escape($buggregator->tfc->getRequest('sort_by','acsending')) ; $sort_order = $buggregator->escape($buggregator->tfc->getRequest('sort_order','acsending')) ; $tool = $buggregator->escape($buggregator->tfc->getRequest('tool','')) ; @@ -38,6 +80,7 @@ } $sql .= " LIMIT {$limit} OFFSET {$offset}" ; $out['sql'] = $sql ; + $out['data']['results'] = [] ; $result = $buggregator->getSQL ( $sql ) ; while($o = $result->fetch_object()) $out['data']['results'][] = $o ; @@ -50,12 +93,7 @@ if($o = $result->fetch_object()) $out['data']['stats']['total_open'] = $o->cnt ; } else if ( $action == 'get_config' ) { - $out['data'] = [ - 'status' => ['OPEN','CLOSED'] , - 'site' => ['WIKI','WIKIDATA','GITHUB','BITBUCKET'] , - 'priority' => ['HIGH','NORMAL','LOW'] , - 'tools' => [] - ] ; + $out['data'] = $config ; $sql = "SELECT * FROM `tool` ORDER BY `name`,`subdomain`" ; $result = $buggregator->getSQL ( $sql ) ; while($o = $result->fetch_object()) { @@ -66,6 +104,22 @@ $out['data']['tools']["{$o->id}"] = $o ; } +} else if ( $action == 'set_issue_status' ) { + + $issue_id = (int) $buggregator->tfc->getRequest('issue_id',0) ; + $new_status = $buggregator->tfc->getRequest('new_status','') ; + if ( !user_can_edit() ) { + $out['status'] = "User not logged in, or not in whitelist" ; + } else if ( $issue_id == 0 ) { + $out['status'] = "No issue ID set" ; + } else if ( !in_array ( $new_status , $config['status']) ) { + $out['status'] = "'{$new_status}' is not a valid issue status" ; + } else { + $buggregator->log ( $issue_id , 'STATUS' , 'CLOSED' , $out['user'] ) ; + $sql = "UPDATE `issue` SET `status`='" . $buggregator->escape($new_status) . "' WHERE `id`={$issue_id}" ; + $buggregator->getSQL($sql); + } + } else { $o->status = "Unknown action '{$action}'" ; } diff --git a/public_html/buggregator/vue.js b/public_html/buggregator/vue.js index 8410eb3..9da231b 100644 --- a/public_html/buggregator/vue.js +++ b/public_html/buggregator/vue.js @@ -4,11 +4,13 @@ let router ; let app ; let wd = new WikiData() ; let config = {} ; +var widar_api_url ; $(document).ready ( function () { + widar_api_url = 'https://magnustools.toolforge.org/buggregator/api.php' ; vue_components.toolname = 'buggregator' ; Promise.all ( [ - vue_components.loadComponents ( ['wd-link','tool-translate','tool-navbar','typeahead-search', + vue_components.loadComponents ( ['wd-link','tool-translate','tool-navbar','typeahead-search','widar', 'vue_components/issue-list.html', ] ) , new Promise(function(resolve, reject) { diff --git a/public_html/buggregator/vue_components/issue-list.html b/public_html/buggregator/vue_components/issue-list.html index 4408dc0..fe70199 100644 --- a/public_html/buggregator/vue_components/issue-list.html +++ b/public_html/buggregator/vue_components/issue-list.html @@ -1,12 +1,9 @@ + + diff --git a/scripts/buggregator/Buggregator.php b/scripts/buggregator/Buggregator.php index 750738e..3451ac8 100644 --- a/scripts/buggregator/Buggregator.php +++ b/scripts/buggregator/Buggregator.php @@ -7,7 +7,7 @@ class Issue { const ZERO_TIME = '0000-00-00 00:00:00' ; - const FIELDS = ['label','status','date_created','date_last','site','url','description','tool','priority'] ; + const FIELDS = ['label','status','date_created','date_last','site','url','description_text_id','tool','priority'] ; protected $issue_id ; protected $label ; @@ -16,6 +16,7 @@ class Issue { protected $date_last = self::ZERO_TIME ; protected $site = 'WIKI' ; protected $url = '' ; + protected $description_text_id = 0 ; protected $description = '' ; protected $tool = 0 ; protected $priority = 'NORMAL' ; @@ -28,6 +29,7 @@ public function date_last() { return $this->date_last ; } public function site() { return $this->site ; } public function url() { return $this->url ; } public function description() { return $this->description ; } + public function description_text_id() { return $this->description_text_id * 1 ; } public function tool() { return $this->tool * 1 ; } public function priority() { return $this->priority ; } @@ -124,6 +126,9 @@ protected function create_as_new_issue ( $buggregator ) { $this->set_times() ; $this->construct_url ( $buggregator ) ; $this->determine_tool ( $buggregator ) ; + + $this->description_text_id = $buggregator->get_or_create_text_id ( $this->description ) ; + $values = [] ; foreach ( self::FIELDS AS $field ) $values[] = $buggregator->escape($this->$field); $fields = implode('`,`',self::FIELDS) ; @@ -450,6 +455,44 @@ static public function format_time ( $time ) { return date('Y-m-d H:i:s',$time); } + public function toolhub_update() { + $sql = "SELECT DISTINCT `toolhub` FROM `tool` WHERE `toolhub`!=''" ; + $known_toolhub = [] ; + $result = $this->getSQL ( $sql ) ; + while($o = $result->fetch_object()) $known_toolhub[$o->toolhub] = $o->toolhub ; + $url = 'https://toolhub-demo.wmcloud.org/api/search/tools/?format=json&ordering=-score&page=1&page_size=1000&q=%22Magnus+Manske%22' ; + $j = json_decode ( file_get_contents($url) ) ; + foreach ( $j->results AS $r ) { + if ( isset($known_toolhub[$r->name]) ) continue ; # We have that + + $candidates = [] ; + + $safe_url = $this->escape(str_replace('http:','https:',$r->url)) ; + $sql = "SELECT * FROM `tool` WHERE `url`='{$safe_url}' AND `toolhub`=''" ; + while($o = $result->fetch_object()) $candidates[$o->id] = $o ; + + $safe_name = $this->escape($r->name) ; + $safe_titles = [] ; + $safe_titles[] = $this->escape($r->title) ; + $safe_titles[] = $this->escape(str_replace(' ','_',$r->title)) ; + $safe_titles[] = $this->escape(str_replace(' ','-',$r->title)) ; + $safe_titles[] = $this->escape(str_replace(' ','',$r->title)) ; + $safe_titles = "'".implode("','",$safe_titles)."'" ; + $sql = "SELECT * FROM `tool` WHERE `name` IN ({$safe_titles}) AND `toolhub`=''" ; + $result = $this->getSQL ( $sql ) ; + while($o = $result->fetch_object()) $candidates[$o->id] = $o ; + $candidates = array_values($candidates) ; + if ( count($candidates) == 0 ) { + print "Not found: {$r->title} / {$r->name} : {$r->url}\n" ; + } else if ( count($candidates) == 1 ) { + $o = $candidates[0] ; + $sql = "UPDATE `tool` SET `toolhub`='{$safe_name}' WHERE `id`={$o->id}" ; + } else { + print "More than one found: {$r->title} / {$r->name} : {$r->url}\n" ; + } + } + } + protected function update_from_wikipages() { $sql = "SELECT * FROM `wiki_page`" ; $result = $this->getSQL ( $sql ) ; @@ -642,7 +685,7 @@ protected function maintenance_wiki_tool_guess () { protected function maintenance_not_wiki_tool_guess () { # WIKI gets special treatment, see above - $sql = "SELECT `id`,`label`,`description` FROM `issue` WHERE `tool`=0 AND `status`='OPEN' AND `site`!='WIKI'" ; + $sql = "SELECT `id`,`label`,`description` FROM `vw_issue` WHERE `tool`=0 AND `status`='OPEN' AND `site`!='WIKI'" ; $result = $this->getSQL ( $sql ) ; while($o = $result->fetch_object()) { $tool_id = Issue::determine_tool_from_text ( $o->label , $this ) ; @@ -681,7 +724,20 @@ protected function maintenance_wiki_close_old_replied () { } } + /* + protected function maintenance_description_text_id() { + $sql = "SELECT * FROM `issue` WHERE `description_text_id`=0" ; + $result = $this->getSQL ( $sql ) ; + while($o = $result->fetch_object()) { + $text_id = $this->get_or_create_text_id ( $o->description ) ; + $sql = "UPDATE `issue` SET `description_text_id`={$text_id} WHERE `id`={$o->id}" ; + $this->getSQL ( $sql ) ; + } + } + */ + public function maintenance () { + #$this->maintenance_description_text_id() ; $this->maintenance_wiki_dates() ; $this->maintenance_wiki_tool_guess() ; $this->maintenance_not_wiki_tool_guess() ; diff --git a/scripts/buggregator/update.php b/scripts/buggregator/update.php index 91493f4..7f25457 100755 --- a/scripts/buggregator/update.php +++ b/scripts/buggregator/update.php @@ -7,6 +7,7 @@ if ( $argv[1] == 'update' ) $buggregator->update() ; else if ( $argv[1] == 'maintenance' ) $buggregator->maintenance() ; +else if ( $argv[1] == 'toolhub' ) $buggregator->toolhub_update() ; else print "Nothing to do!\n" ; ?> \ No newline at end of file From b924296350aab930b8026afc7ff8760dd435d8d4 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Thu, 21 Oct 2021 10:43:53 +0000 Subject: [PATCH 16/45] auto-generation of toolinfo --- public_html/resources/vue/wikidatamap.html | 45 +++++++++--- scripts/buggregator/Buggregator.php | 7 +- scripts/buggregator/buggregator2toolinfo.php | 76 ++++++++++++++++++++ 3 files changed, 117 insertions(+), 11 deletions(-) create mode 100755 scripts/buggregator/buggregator2toolinfo.php diff --git a/public_html/resources/vue/wikidatamap.html b/public_html/resources/vue/wikidatamap.html index 9a6c315..bf3c62e 100644 --- a/public_html/resources/vue/wikidatamap.html +++ b/public_html/resources/vue/wikidatamap.html @@ -2,7 +2,8 @@ @@ -76,8 +79,8 @@ Vue.component ( 'wd-link' , { template : '#wd-link-template' , - props : [ 'item' , 'wd' , 'label' , 'as_text' , 'smallq' , 'language' ] , - data : function () { return { use_label:'' , wikidata:{} , url:'' } } , + props : [ 'item' , 'wd' , 'label' , 'as_text' , 'smallq' , 'language' , 'string_prop' ] , + data : function () { return { use_label:'' , wikidata:{} , url:'' , string_prop_value:'' } } , created : function () { let me = this ; if ( typeof me.wd == 'undefined' ) me.wikidata = wd_link_wd ; @@ -96,6 +99,9 @@ if ( typeof i == 'undefined' ) return ; if ( typeof me.language == 'undefined' ) me.use_label = i.getLabel () ; else me.use_label = i.getLabel ( me.language ) ; + if ( typeof me.string_prop!="undefined" ) { + me.string_prop_value = i.getStringsForProperty(me.string_prop).join('; ') ; + } } ) ; } , methods : { diff --git a/public_html/resources/vue/widar.html b/public_html/resources/vue/widar.html index 4162ceb..064e5aa 100644 --- a/public_html/resources/vue/widar.html +++ b/public_html/resources/vue/widar.html @@ -15,7 +15,7 @@ var widar_api_url ; Vue.component ( 'widar' , { - data : function () { return { is_logged_in:false , userinfo:{} , widar_api:'/widar/index.php' , loaded:false , toolname:'' } } , + data : function () { return { is_logged_in:false , userinfo:{} , widar_api:'/widar/index.php' , loaded:false , toolname:'' , maximum_number_of_tries:-1 , last_error:'' } } , created : function () { widar = this ; if ( typeof widar_api_url != 'undefined' ) this.widar_api = widar_api_url ; @@ -52,16 +52,22 @@ } ) */ } , - run : function ( params , callback ) { + run : function ( params , callback , number_of_tries = 0 ) { let me = this ; + if ( me.maximum_number_of_tries>0 && number_of_tries>me.maximum_number_of_tries ) { + console.log ( "Aborting after "+number_of_tries+" attempts") ; + console.log ( me.last_error ) ; + return callback() ; + } if ( me.tool != '' && typeof params.tool_hashtag == 'undefined' ) params.tool_hashtag = me.toolname ; params.botmode = 1 ; $.post ( me.widar_api , params , function ( d ) { if ( d.error != 'OK' ) { //console.log ( 'WIDAR' , params ) ; if ( null != d.error.match(/Invalid token/) || null != d.error.match(/happen/) || null != d.error.match(/Problem creating item/) || ( params.action!='create_redirect' && null != d.error.match(/failed/) ) ) { + me.last_error = d.error ; console.log ( "ERROR (re-trying)" , params , d ) ; - setTimeout ( function () { me.run ( params , callback ) } , 500 ) ; // Again + setTimeout ( function () { me.run ( params , callback , number_of_tries+1 ) } , 500 ) ; // Again } else { console.log ( "ERROR (aborting)" , params , d ) ; // let h = "
  • ERROR (" + params.action + ") : " + d.error + "
    • " ; @@ -73,7 +79,8 @@ } } , 'json' ) . fail(function() { console.log ( "Again" , params ) ; - me.run ( params , callback ) ; + me.last_error = "POST fail" ; + me.run ( params , callback , number_of_tries+1 ) ; }) ; } , newClaimDate : function ( prop , d ) { diff --git a/scripts/logger/Logger.php b/scripts/logger/Logger.php index e9a25be..56eebc8 100644 --- a/scripts/logger/Logger.php +++ b/scripts/logger/Logger.php @@ -5,20 +5,20 @@ class Logger { public $tfc ; public $db ; - private $tools_table_name = 'tools' ; - private $logs_table_name = 'logs' ; + private $database_name = 'buggregator' ; + private $tools_table_name = 'logging_tools' ; + private $logs_table_name = 'logging_uses' ; public function __construct () { - $this->tfc = new ToolforgeCommon('logger') ; - $this->db = $this->tfc->openDBtool ( 'tool_logging' ) ; + $this->tfc = new ToolforgeCommon ( 'logger' ) ; + $this->db = $this->tfc->openDBtool ( $this->database_name ) ; } - public function getToolByNameAndMethod ( $toolname , $method ) { - $this->sanitizeToolAndMethodName ( $toolname , $method ) ; + public function getToolByNameAndMethod ( $toolname , $method , $sanitize = true ) { + if ( $sanitize ) $this->sanitizeToolAndMethodName ( $toolname , $method ) ; $sql = "SELECT id FROM `{$this->tools_table_name}` WHERE `name`='" . $this->db->real_escape_string($toolname) . "' AND `method`='" . $this->db->real_escape_string($method) . "'" ; $result = $this->tfc->getSQL ( $this->db , $sql ) ; - while($o = $result->fetch_object()) $tool_id = $o->id ; - return $tool_id ; + while($o = $result->fetch_object()) return $o->id ; } protected function createNewToolAndMethod ( $toolname , $method ) { @@ -45,10 +45,10 @@ public function deleteToolById ( $tool_id ) { $this->tfc->getSQL ( $this->db , $sql ) ; } - public function addToLog ( $tool_id , $increase = 1 ) { + public function addToLog ( $tool_id , $date = 0 , $increase = 1 ) { # Create tool/date entry if necessary, increase usage count - $date = $this->today() ; - $sql = "INSERT INTO `{$this->logs_table_name}` (`tool_id`,`date`,`used`) VALUES ({$tool_id},{$date},1) ON DUPLICATE KEY UPDATE `used`=`used`+{$increase}" ; + if ( $date == 0 ) $date = $this->today() ; + $sql = "INSERT INTO `{$this->logs_table_name}` (`tool_id`,`date`,`used`) VALUES ({$tool_id},{$date},{$increase}) ON DUPLICATE KEY UPDATE `used`=`used`+{$increase}" ; $this->tfc->getSQL ( $this->db , $sql ) ; } @@ -73,13 +73,38 @@ public function micDrop ( $error = '' ) { exit(0) ; } - protected function sanitizeToolAndMethodName ( &$toolname , &$method ) { - $method = preg_replace ( '|\s*[\(\)].*$|' , '' , $method ) ; + public function sanitizeToolAndMethodName ( &$toolname , &$method ) { + $method = preg_replace ( '|\s*[\(\)\"\'\=\;\,].*$|' , '' , $method ) ; + if ( $toolname == 'mix-n-match' ) { + $method = preg_replace ( '/\s+(and|union)\b.*/i' , '' , $method ) ; + $method = preg_replace ( '|\..*|i' , '' , $method ) ; + } + if ( $toolname == 'icommons' ) { + if ( preg_match('|^category\b|',$method) ) $method = 'category' ; + } $toolname = trim ( strtolower ( $toolname ) ) ; $method = trim ( strtolower ( $method ) ) ; if ( $toolname == '' ) $logger->micDrop ( "No tool name supplied" ) ; } + public function mergeMethods ( $toolname , $method_from , $method_to ) { + $tool_id_from = $this->getToolByNameAndMethod ( $toolname , $method_from , false ) ; + if ( !isset($tool_id_from) ) return $this->micDrop ( "No tool {$toolname}:{$method_from}" ) ; + $tool_id_to = $this->getOrCreateToolByNameAndMethod ( $toolname , $method_to ) ; + $sql = "SELECT * FROM `{$this->logs_table_name}` WHERE `tool_id`={$tool_id_from}" ; + $result = $this->tfc->getSQL ( $this->db , $sql ) ; + #print "{$toolname}: {$method_from}={$tool_id_from} => {$method_to}={$tool_id_to}\n" ; + # Merge and delete logs + while($o = $result->fetch_object()) { + $this->addToLog ( $tool_id_to , $o->date , $o->used ) ; + $sql = "DELETE FROM `{$this->logs_table_name}` WHERE `id`={$o->id}" ; + $this->tfc->getSQL ( $this->db , $sql ) ; + } + # Delete tool entry + $sql = "DELETE FROM `{$this->tools_table_name}` WHERE `id`={$tool_id_from}" ; + $this->tfc->getSQL ( $this->db , $sql ) ; + } + protected function today() { $date = date ( 'Ymd' ) * 1 ; return $date ; diff --git a/scripts/logger/run_logger.php b/scripts/logger/run_logger.php index 5ab0621..8664ea4 100755 --- a/scripts/logger/run_logger.php +++ b/scripts/logger/run_logger.php @@ -5,13 +5,28 @@ $logger = new Logger ; -$tool_id = $logger->getOrCreateToolByNameAndMethod('testing','misc requests') ; -print "Tool {$tool_id}\n" ; -$logger->addToLog ( $tool_id ) ; -$logger->addToLog ( $tool_id ) ; -$uses = $logger->getAllUses ( $tool_id ) ; -print "Uses {$uses}, expecting 2\n" ; -$logger->deleteToolById($tool_id) ; -$logger->micDrop(); +if ( false ) { + # Merge methods with funny names + $sql = "SELECT * FROM tools" ; + $result = $logger->tfc->getSQL ( $logger->db , $sql ) ; + while($o = $result->fetch_object()) { + $name = "{$o->name}" ; + $method = "{$o->method}" ; + $logger->sanitizeToolAndMethodName ( $name , $method ) ; + if ( $name==$o->name and $method==$o->method ) continue ; + $logger->mergeMethods ( $o->name , $o->method , $method ) ; + } + +} else { + + $tool_id = $logger->getOrCreateToolByNameAndMethod('testing','misc requests') ; + print "Tool {$tool_id}\n" ; + $logger->addToLog ( $tool_id ) ; + $logger->addToLog ( $tool_id ) ; + $uses = $logger->getAllUses ( $tool_id ) ; + print "Uses {$uses}, expecting 2\n" ; + $logger->deleteToolById($tool_id) ; + $logger->micDrop(); +} ?> \ No newline at end of file diff --git a/tjs b/tjs new file mode 100644 index 0000000..14ad181 --- /dev/null +++ b/tjs @@ -0,0 +1,4 @@ +toolforge-jobs run --image tf-php74 --schedule '9 9 * * *' --mem 100Mi --command 'find /data/project/magnustools/tmp/ -mtime +1 -exec rm -rf {} \;' cleanup-tmp +toolforge-jobs run --image tf-php74 --schedule '37 3 * * *' --mem 2Gi --command '/data/project/magnustools/scripts/buggregator/update.php update' buggregator-update +toolforge-jobs run --image tf-php74 --schedule '37 4 * * *' --mem 2Gi --command '/data/project/magnustools/scripts/buggregatorbuggregator2toolinfo.php' buggregator-toolinfo + From 205074b4ee38aefdf6f25b1ef787bab7d0868971 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Wed, 30 Nov 2022 11:02:21 +0000 Subject: [PATCH 25/45] bot v1 --- composer.json | 3 +- composer.lock | 820 ++++++++++++++---- public_html/html2wiki.php | 8 +- .../circular_redirects/circular_redirects.php | 188 ++++ 4 files changed, 863 insertions(+), 156 deletions(-) create mode 100755 scripts/circular_redirects/circular_redirects.php diff --git a/composer.json b/composer.json index 9d27b2a..2389082 100644 --- a/composer.json +++ b/composer.json @@ -28,6 +28,7 @@ "require": { "php": ">=7.0.0", "abraham/twitteroauth": "^2.0", - "mediawiki/oauthclient": "^1.2" + "mediawiki/oauthclient": "^1.2", + "addwiki/mediawiki-api": "^2.8" } } diff --git a/composer.lock b/composer.lock index a5151e2..1d8fbc9 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "5802849254b75d7e4d4945b47161c62c", + "content-hash": "7bf739a83aa8f975181506eb616a094b", "packages": [ { "name": "abraham/twitteroauth", @@ -63,6 +63,161 @@ ], "time": "2020-12-02T01:27:06+00:00" }, + { + "name": "addwiki/mediawiki-api", + "version": "2.8.0", + "source": { + "type": "git", + "url": "https://github.com/addwiki/mediawiki-api.git", + "reference": "eee8f072aba0a2e9886f290a73b8e10eb83e0cfc" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/addwiki/mediawiki-api/zipball/eee8f072aba0a2e9886f290a73b8e10eb83e0cfc", + "reference": "eee8f072aba0a2e9886f290a73b8e10eb83e0cfc", + "shasum": "" + }, + "require": { + "addwiki/mediawiki-api-base": "^2.8", + "addwiki/mediawiki-datamodel": "^2.8", + "php": ">=7.3" + }, + "require-dev": { + "mediawiki/mediawiki-codesniffer": "~35.0", + "monolog/monolog": "^1.23", + "php-parallel-lint/php-parallel-lint": "^1.2", + "phpunit/phpunit": "~9" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "2.8-dev" + } + }, + "autoload": { + "psr-4": { + "Mediawiki\\Api\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "GPL-2.0-or-later" + ], + "authors": [ + { + "name": "Addshore" + } + ], + "description": "A MediaWiki API library", + "keywords": [ + "api", + "mediawiki" + ], + "time": "2021-02-16T19:46:17+00:00" + }, + { + "name": "addwiki/mediawiki-api-base", + "version": "2.8.0", + "source": { + "type": "git", + "url": "https://github.com/addwiki/mediawiki-api-base.git", + "reference": "d8eff31b54fd39d90eb14457e5ce478cfdf08394" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/addwiki/mediawiki-api-base/zipball/d8eff31b54fd39d90eb14457e5ce478cfdf08394", + "reference": "d8eff31b54fd39d90eb14457e5ce478cfdf08394", + "shasum": "" + }, + "require": { + "guzzlehttp/guzzle": "~6.3|~7.0", + "guzzlehttp/promises": "~1.0", + "php": ">=7.3", + "psr/log": "~1.0" + }, + "require-dev": { + "mediawiki/mediawiki-codesniffer": "~35.0", + "php-parallel-lint/php-parallel-lint": "^1.2", + "phpunit/phpunit": "~9" + }, + "suggest": { + "etsy/phan": "Allows running static analysis on the package (requires PHP 7+)" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "2.8-dev" + } + }, + "autoload": { + "psr-4": { + "Mediawiki\\Api\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "GPL-2.0-or-later" + ], + "authors": [ + { + "name": "Addshore" + } + ], + "description": "Simple MediaWiki API library", + "keywords": [ + "api", + "mediawiki" + ], + "time": "2021-02-16T19:46:17+00:00" + }, + { + "name": "addwiki/mediawiki-datamodel", + "version": "2.8.0", + "source": { + "type": "git", + "url": "https://github.com/addwiki/mediawiki-datamodel.git", + "reference": "1082c8cd947ca313d4f2b95f9b1dc2b2b492052a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/addwiki/mediawiki-datamodel/zipball/1082c8cd947ca313d4f2b95f9b1dc2b2b492052a", + "reference": "1082c8cd947ca313d4f2b95f9b1dc2b2b492052a", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "require-dev": { + "mediawiki/mediawiki-codesniffer": "~35.0", + "php-parallel-lint/php-parallel-lint": "^1.2", + "phpunit/phpunit": "~9" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "2.8-dev" + } + }, + "autoload": { + "psr-4": { + "Mediawiki\\DataModel\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "GPL-2.0-or-later" + ], + "authors": [ + { + "name": "Addshore" + } + ], + "description": "A Mediawiki datamodel", + "keywords": [ + "mediawiki" + ], + "time": "2021-02-16T19:46:19+00:00" + }, { "name": "composer/ca-bundle", "version": "1.3.1", @@ -118,21 +273,284 @@ "ssl", "tls" ], - "funding": [ + "time": "2021-10-28T20:44:15+00:00" + }, + { + "name": "guzzlehttp/guzzle", + "version": "7.5.0", + "source": { + "type": "git", + "url": "https://github.com/guzzle/guzzle.git", + "reference": "b50a2a1251152e43f6a37f0fa053e730a67d25ba" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/b50a2a1251152e43f6a37f0fa053e730a67d25ba", + "reference": "b50a2a1251152e43f6a37f0fa053e730a67d25ba", + "shasum": "" + }, + "require": { + "ext-json": "*", + "guzzlehttp/promises": "^1.5", + "guzzlehttp/psr7": "^1.9 || ^2.4", + "php": "^7.2.5 || ^8.0", + "psr/http-client": "^1.0", + "symfony/deprecation-contracts": "^2.2 || ^3.0" + }, + "provide": { + "psr/http-client-implementation": "1.0" + }, + "require-dev": { + "bamarni/composer-bin-plugin": "^1.8.1", + "ext-curl": "*", + "php-http/client-integration-tests": "^3.0", + "phpunit/phpunit": "^8.5.29 || ^9.5.23", + "psr/log": "^1.1 || ^2.0 || ^3.0" + }, + "suggest": { + "ext-curl": "Required for CURL handler support", + "ext-intl": "Required for Internationalized Domain Name (IDN) support", + "psr/log": "Required for using the Log middleware" + }, + "type": "library", + "extra": { + "bamarni-bin": { + "bin-links": true, + "forward-command": false + }, + "branch-alias": { + "dev-master": "7.5-dev" + } + }, + "autoload": { + "files": [ + "src/functions_include.php" + ], + "psr-4": { + "GuzzleHttp\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Graham Campbell", + "email": "hello@gjcampbell.co.uk", + "homepage": "https://github.com/GrahamCampbell" + }, + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + }, + { + "name": "Jeremy Lindblom", + "email": "jeremeamia@gmail.com", + "homepage": "https://github.com/jeremeamia" + }, { - "url": "https://packagist.com", - "type": "custom" + "name": "George Mponos", + "email": "gmponos@gmail.com", + "homepage": "https://github.com/gmponos" }, { - "url": "https://github.com/composer", - "type": "github" + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com", + "homepage": "https://github.com/Nyholm" }, { - "url": "https://tidelift.com/funding/github/packagist/composer/composer", - "type": "tidelift" + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com", + "homepage": "https://github.com/sagikazarmark" + }, + { + "name": "Tobias Schultze", + "email": "webmaster@tubo-world.de", + "homepage": "https://github.com/Tobion" } ], - "time": "2021-10-28T20:44:15+00:00" + "description": "Guzzle is a PHP HTTP client library", + "keywords": [ + "client", + "curl", + "framework", + "http", + "http client", + "psr-18", + "psr-7", + "rest", + "web service" + ], + "time": "2022-08-28T15:39:27+00:00" + }, + { + "name": "guzzlehttp/promises", + "version": "1.5.2", + "source": { + "type": "git", + "url": "https://github.com/guzzle/promises.git", + "reference": "b94b2807d85443f9719887892882d0329d1e2598" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/promises/zipball/b94b2807d85443f9719887892882d0329d1e2598", + "reference": "b94b2807d85443f9719887892882d0329d1e2598", + "shasum": "" + }, + "require": { + "php": ">=5.5" + }, + "require-dev": { + "symfony/phpunit-bridge": "^4.4 || ^5.1" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.5-dev" + } + }, + "autoload": { + "files": [ + "src/functions_include.php" + ], + "psr-4": { + "GuzzleHttp\\Promise\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Graham Campbell", + "email": "hello@gjcampbell.co.uk", + "homepage": "https://github.com/GrahamCampbell" + }, + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + }, + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com", + "homepage": "https://github.com/Nyholm" + }, + { + "name": "Tobias Schultze", + "email": "webmaster@tubo-world.de", + "homepage": "https://github.com/Tobion" + } + ], + "description": "Guzzle promises library", + "keywords": [ + "promise" + ], + "time": "2022-08-28T14:55:35+00:00" + }, + { + "name": "guzzlehttp/psr7", + "version": "2.4.3", + "source": { + "type": "git", + "url": "https://github.com/guzzle/psr7.git", + "reference": "67c26b443f348a51926030c83481b85718457d3d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/67c26b443f348a51926030c83481b85718457d3d", + "reference": "67c26b443f348a51926030c83481b85718457d3d", + "shasum": "" + }, + "require": { + "php": "^7.2.5 || ^8.0", + "psr/http-factory": "^1.0", + "psr/http-message": "^1.0", + "ralouphie/getallheaders": "^3.0" + }, + "provide": { + "psr/http-factory-implementation": "1.0", + "psr/http-message-implementation": "1.0" + }, + "require-dev": { + "bamarni/composer-bin-plugin": "^1.8.1", + "http-interop/http-factory-tests": "^0.9", + "phpunit/phpunit": "^8.5.29 || ^9.5.23" + }, + "suggest": { + "laminas/laminas-httphandlerrunner": "Emit PSR-7 responses" + }, + "type": "library", + "extra": { + "bamarni-bin": { + "bin-links": true, + "forward-command": false + }, + "branch-alias": { + "dev-master": "2.4-dev" + } + }, + "autoload": { + "psr-4": { + "GuzzleHttp\\Psr7\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Graham Campbell", + "email": "hello@gjcampbell.co.uk", + "homepage": "https://github.com/GrahamCampbell" + }, + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + }, + { + "name": "George Mponos", + "email": "gmponos@gmail.com", + "homepage": "https://github.com/gmponos" + }, + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com", + "homepage": "https://github.com/Nyholm" + }, + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com", + "homepage": "https://github.com/sagikazarmark" + }, + { + "name": "Tobias Schultze", + "email": "webmaster@tubo-world.de", + "homepage": "https://github.com/Tobion" + }, + { + "name": "Márk Sági-Kazár", + "email": "mark.sagikazar@gmail.com", + "homepage": "https://sagikazarmark.hu" + } + ], + "description": "PSR-7 message implementation that also provides common utility methods", + "keywords": [ + "http", + "message", + "psr-7", + "request", + "response", + "stream", + "uri", + "url" + ], + "time": "2022-10-26T14:07:24+00:00" }, { "name": "mediawiki/oauthclient", @@ -189,6 +607,157 @@ "homepage": "https://www.mediawiki.org/wiki/oauthclient-php", "time": "2021-10-21T13:34:48+00:00" }, + { + "name": "psr/http-client", + "version": "1.0.1", + "source": { + "type": "git", + "url": "https://github.com/php-fig/http-client.git", + "reference": "2dfb5f6c5eff0e91e20e913f8c5452ed95b86621" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/http-client/zipball/2dfb5f6c5eff0e91e20e913f8c5452ed95b86621", + "reference": "2dfb5f6c5eff0e91e20e913f8c5452ed95b86621", + "shasum": "" + }, + "require": { + "php": "^7.0 || ^8.0", + "psr/http-message": "^1.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Http\\Client\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "http://www.php-fig.org/" + } + ], + "description": "Common interface for HTTP clients", + "homepage": "https://github.com/php-fig/http-client", + "keywords": [ + "http", + "http-client", + "psr", + "psr-18" + ], + "time": "2020-06-29T06:28:15+00:00" + }, + { + "name": "psr/http-factory", + "version": "1.0.1", + "source": { + "type": "git", + "url": "https://github.com/php-fig/http-factory.git", + "reference": "12ac7fcd07e5b077433f5f2bee95b3a771bf61be" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/http-factory/zipball/12ac7fcd07e5b077433f5f2bee95b3a771bf61be", + "reference": "12ac7fcd07e5b077433f5f2bee95b3a771bf61be", + "shasum": "" + }, + "require": { + "php": ">=7.0.0", + "psr/http-message": "^1.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Http\\Message\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "http://www.php-fig.org/" + } + ], + "description": "Common interfaces for PSR-7 HTTP message factories", + "keywords": [ + "factory", + "http", + "message", + "psr", + "psr-17", + "psr-7", + "request", + "response" + ], + "time": "2019-04-30T12:38:16+00:00" + }, + { + "name": "psr/http-message", + "version": "1.0.1", + "source": { + "type": "git", + "url": "https://github.com/php-fig/http-message.git", + "reference": "f6561bf28d520154e4b0ec72be95418abe6d9363" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/http-message/zipball/f6561bf28d520154e4b0ec72be95418abe6d9363", + "reference": "f6561bf28d520154e4b0ec72be95418abe6d9363", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Http\\Message\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "http://www.php-fig.org/" + } + ], + "description": "Common interface for HTTP messages", + "homepage": "https://github.com/php-fig/http-message", + "keywords": [ + "http", + "http-message", + "psr", + "psr-7", + "request", + "response" + ], + "time": "2016-08-06T14:39:51+00:00" + }, { "name": "psr/log", "version": "1.1.4", @@ -235,6 +804,96 @@ "psr-3" ], "time": "2021-05-03T11:20:27+00:00" + }, + { + "name": "ralouphie/getallheaders", + "version": "3.0.3", + "source": { + "type": "git", + "url": "https://github.com/ralouphie/getallheaders.git", + "reference": "120b605dfeb996808c31b6477290a714d356e822" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/ralouphie/getallheaders/zipball/120b605dfeb996808c31b6477290a714d356e822", + "reference": "120b605dfeb996808c31b6477290a714d356e822", + "shasum": "" + }, + "require": { + "php": ">=5.6" + }, + "require-dev": { + "php-coveralls/php-coveralls": "^2.1", + "phpunit/phpunit": "^5 || ^6.5" + }, + "type": "library", + "autoload": { + "files": [ + "src/getallheaders.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ralph Khattar", + "email": "ralph.khattar@gmail.com" + } + ], + "description": "A polyfill for getallheaders.", + "time": "2019-03-08T08:55:37+00:00" + }, + { + "name": "symfony/deprecation-contracts", + "version": "v2.5.2", + "source": { + "type": "git", + "url": "https://github.com/symfony/deprecation-contracts.git", + "reference": "e8b495ea28c1d97b5e0c121748d6f9b53d075c66" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/e8b495ea28c1d97b5e0c121748d6f9b53d075c66", + "reference": "e8b495ea28c1d97b5e0c121748d6f9b53d075c66", + "shasum": "" + }, + "require": { + "php": ">=7.1" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "2.5-dev" + }, + "thanks": { + "name": "symfony/contracts", + "url": "https://github.com/symfony/contracts" + } + }, + "autoload": { + "files": [ + "function.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "A generic function and convention to trigger deprecation notices", + "homepage": "https://symfony.com", + "time": "2022-01-02T09:53:40+00:00" } ], "packages-dev": [ @@ -287,20 +946,6 @@ "constructor", "instantiate" ], - "funding": [ - { - "url": "https://www.doctrine-project.org/sponsorship.html", - "type": "custom" - }, - { - "url": "https://www.patreon.com/phpdoctrine", - "type": "patreon" - }, - { - "url": "https://tidelift.com/funding/github/packagist/doctrine%2Finstantiator", - "type": "tidelift" - } - ], "time": "2020-11-10T18:47:58+00:00" }, { @@ -349,12 +994,6 @@ "object", "object graph" ], - "funding": [ - { - "url": "https://tidelift.com/funding/github/packagist/myclabs/deep-copy", - "type": "tidelift" - } - ], "time": "2020-11-13T09:40:50+00:00" }, { @@ -732,12 +1371,6 @@ "testing", "xunit" ], - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2021-07-26T12:20:09+00:00" }, { @@ -788,12 +1421,6 @@ "filesystem", "iterator" ], - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2021-07-19T06:46:01+00:00" }, { @@ -884,12 +1511,6 @@ "keywords": [ "timer" ], - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2020-11-30T08:20:02+00:00" }, { @@ -939,12 +1560,6 @@ "keywords": [ "tokenizer" ], - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "abandoned": true, "time": "2021-07-26T12:15:06+00:00" }, @@ -1029,16 +1644,6 @@ "testing", "xunit" ], - "funding": [ - { - "url": "https://phpunit.de/donate.html", - "type": "custom" - }, - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2021-09-25T07:37:20+00:00" }, { @@ -1084,12 +1689,6 @@ ], "description": "Looks up which function or method a line of code belongs to", "homepage": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/", - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2020-11-30T08:15:22+00:00" }, { @@ -1154,12 +1753,6 @@ "compare", "equality" ], - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2020-11-30T08:04:30+00:00" }, { @@ -1216,12 +1809,6 @@ "unidiff", "unified diff" ], - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2020-11-30T07:59:04+00:00" }, { @@ -1275,12 +1862,6 @@ "environment", "hhvm" ], - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2020-11-30T07:53:42+00:00" }, { @@ -1348,12 +1929,6 @@ "export", "exporter" ], - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2020-11-30T07:47:53+00:00" }, { @@ -1408,12 +1983,6 @@ "keywords": [ "global state" ], - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2020-11-30T07:43:24+00:00" }, { @@ -1461,12 +2030,6 @@ ], "description": "Traverses array structures and object graphs to enumerate all referenced objects", "homepage": "https://github.com/sebastianbergmann/object-enumerator/", - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2020-11-30T07:40:27+00:00" }, { @@ -1512,12 +2075,6 @@ ], "description": "Allows reflection of object attributes, including inherited and non-public ones", "homepage": "https://github.com/sebastianbergmann/object-reflector/", - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2020-11-30T07:37:18+00:00" }, { @@ -1571,12 +2128,6 @@ ], "description": "Provides functionality to recursively process PHP variables", "homepage": "http://www.github.com/sebastianbergmann/recursion-context", - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2020-11-30T07:34:24+00:00" }, { @@ -1619,12 +2170,6 @@ ], "description": "Provides a list of PHP built-in functions that operate on resources", "homepage": "https://www.github.com/sebastianbergmann/resource-operations", - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2020-11-30T07:30:19+00:00" }, { @@ -1671,12 +2216,6 @@ ], "description": "Collection of value objects that represent the types of the PHP type system", "homepage": "https://github.com/sebastianbergmann/type", - "funding": [ - { - "url": "https://github.com/sebastianbergmann", - "type": "github" - } - ], "time": "2020-11-30T07:25:11+00:00" }, { @@ -1782,20 +2321,6 @@ "polyfill", "portable" ], - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], "time": "2021-02-19T12:13:01+00:00" }, { @@ -1836,12 +2361,6 @@ } ], "description": "A small library for converting tokenized PHP source code into XML and potentially other formats", - "funding": [ - { - "url": "https://github.com/theseer", - "type": "github" - } - ], "time": "2021-07-28T10:34:58+00:00" }, { @@ -1907,6 +2426,5 @@ "platform": { "php": ">=7.0.0" }, - "platform-dev": [], - "plugin-api-version": "1.1.0" + "platform-dev": [] } diff --git a/public_html/html2wiki.php b/public_html/html2wiki.php index 4baab84..6cf123a 100755 --- a/public_html/html2wiki.php +++ b/public_html/html2wiki.php @@ -50,11 +50,11 @@ function html2wiki_tables($str, $row_delim = 1, $oneline = false, $escape = fals '/<\/th>/i', # headers end '/<\/td>/i', # cells end # e - replacement string gets evaluated before the replacement - '/]*)>/ie', # table start + '/]*)>/i', # table start '//i', # caption start - '//Uie', # row start - '//Uie', # header start - '//Uie', # cell start + '//Ui', # row start + '//Ui', # header start + '//Ui', # cell start "/\n$my_nl/", "/$my_nl/", "/\n */", # spaces at beginning of a line diff --git a/scripts/circular_redirects/circular_redirects.php b/scripts/circular_redirects/circular_redirects.php new file mode 100755 index 0000000..206458e --- /dev/null +++ b/scripts/circular_redirects/circular_redirects.php @@ -0,0 +1,188 @@ +#!/usr/bin/php +tfc = new ToolforgeCommon ( 'circular_redirects' ) ; + + $ini_array = parse_ini_file('/data/project/magnustools/circularredirectsbot.ini'); + $this->bot_user_name = $ini_array['name'] ; + $this->bot_user_password = $ini_array['pass'] ; + } + + protected function get_link_pattern($rtitle) { + $pattern = preg_quote($rtitle); + $pattern = preg_replace('/[ _]/','[ _]',$pattern); + if ( ucfirst($rtitle)!=lcfirst($rtitle) ) { + $first_letter_up = ucfirst($rtitle)[0]; + $first_letter_low = lcfirst($rtitle)[0]; + $pattern = preg_replace("/^./","[{$first_letter_up}{$first_letter_low}]",$pattern); + } + $pattern = "/\[\[\s*({$pattern})\s*(|\|.*?)\]\]/"; + return $pattern; + } + + public function update_db($wiki) { + $dbt = $this->tfc->openDBtool("circular_redirects_p"); + $dbw = $this->tfc->openDBwiki($wiki); + $total_count = 0 ; + $offset = 0 ; + $batch_size = 1000 ; + $limit = 10000 ; + $cache = [] ; + while ( true ) { + #print "{$offset}\n" ; + $sql = "SELECT p1.page_title FROM page p1,pagelinks + WHERE p1.page_id=pl_from AND pl_namespace=0 AND pl_from_namespace=0 + AND EXISTS (SELECT * FROM page p2,redirect WHERE p2.page_namespace=0 + AND p2.page_title=pl_title AND rd_from=p2.page_id AND rd_namespace=0 AND rd_title=p1.page_title) + LIMIT {$limit} OFFSET {$offset}"; + $result = $this->tfc->getSQL ( $dbw , $sql ) ; + $old_total = $total_count ; + while($o = $result->fetch_object()){ + $total_count++ ; + $page = $dbt->real_escape_string ( $o->page_title ) ; + $cache[$page] = "('{$wiki}','{$page}')"; + if ( count($cache)>$batch_size) $this->flushUpdate($dbt,$cache); + $offset++; + } + if ( $old_total+$batch_size > $total_count ) break ; + } + $this->flushUpdate($dbt,$cache); + $ts = $this->tfc->getCurrentTimestamp(); + $ts = substr ( $ts , 0 , 8 ) ; # Day + $sql = "INSERT IGNORE INTO `stats` (`wiki`,`date`,`total`) VALUES ('{$wiki}','{$ts}',{$total_count})" ; + $this->tfc->getSQL ( $dbt , $sql ) ; + } + + protected function flushUpdate(&$dbt,&$cache) { + if ( count($cache)==0 ) return ; + $sql = "INSERT IGNORE INTO `pages` (`wiki`,`page`) VALUES ".implode(',',$cache) ; + $this->tfc->getSQL ( $dbt , $sql ) ; + $cache = [] ; + } + + public function simple_fix($wiki,$page,$redirects) { + if ( count($redirects) == 0 ) return ; + $page_nice = ucfirst(preg_replace('/_/',' ',$page)); + $wt = $this->tfc->getWikiPageText($wiki,$page); + $wt_orig = $wt ; + foreach ( $redirects AS $rtitle ) { + if ( preg_match("/^[A-Za-z][a-z]+:/",$rtitle) ) continue ; # TODO skipping redirects with namespaces + $pattern = $this->get_link_pattern($rtitle); + if ( !preg_match_all($pattern,$wt,$matches,PREG_SET_ORDER|PREG_OFFSET_CAPTURE) ) continue ; + while (count($matches)>0) { + $m = array_pop($matches); + $start = $m[0][1]*1; + $end = $start + strlen($m[0][0]); + if ( strlen($m[2][0])>0 ) { # Replace with link text + $wt = substr($wt,0,$start).$m[2][0].substr($wt,$end); + } else { # Replace with linked page + $wt = substr($wt,0,$start).$m[1][0].substr($wt,$end); + } + } + } + if ( $wt!=$wt_orig ) { + $this->setPageText($wiki,$page,$wt,"Replacing circular redirect link (redirect links back to this page) with plain text"); + $ret = true ; + } else { + $ret = false ; + } + return $ret ; + } + + protected function loginToWiki ( $api_url , $wiki_user , $wiki_pass ) { + if ( isset($this->wiki_logins[$api_url]) ) return ; + + $api = new \Mediawiki\Api\MediawikiApi( $api_url ); + $api->login( new \Mediawiki\Api\ApiUser( $wiki_user, $wiki_pass ) ); + $services = new \Mediawiki\Api\MediawikiFactory( $api ); + + $this->wiki_logins[$api_url] = (object) [ 'api' => $api , 'services' => $services ] ; + } + + protected function getApiUrl($wiki) { + $server = $this->tfc->getWebserverForWiki ( $wiki ) ; + $api_url = "https://{$server}/w/api.php" ; + return $api_url; + } + + protected function getWikiServices($wiki) { + $api_url = $this->getApiUrl($wiki); + $this->loginToWiki($api_url,$this->bot_user_name,$this->bot_user_password); + if ( !isset($this->wiki_logins[$api_url]) ) throw new Exception(__METHOD__.": Not logged in to {$api_url}" ) ; + $services = $this->wiki_logins[$api_url]->services ; + return $services; + } + + protected function setPageText ( $wiki , $page_title , $new_wikitext , $summary = '' ) { + $services = $this->getWikiServices($wiki); + $content = new \Mediawiki\DataModel\Content( $new_wikitext ); + $page_title = str_replace(' ','_',$page_title) ; + + $page = $services->newPageGetter()->getFromTitle( $page_title ); + $revisions = (array) $page->getRevisions() ; + $revision = array_pop ( $revisions ) ; + $revision = array_pop ( $revision ) ; + if ( $revision == null ) throw new Exception(__METHOD__.": Not creating new page" ) ; + + $old_wikitext = $revision->getContent()->getData() ; + if ( trim($old_wikitext) == trim($new_wikitext) ) return ; # No change, no edit + + $ei = new \Mediawiki\DataModel\EditInfo($summary,\Mediawiki\DataModel\EditInfo::MINOR); + $revision = new \Mediawiki\DataModel\Revision( $content, $page->getPageIdentifier() , null , $ei ); + $services->newRevisionSaver()->save( $revision ); + } + + public function fix_next_page($wiki) { + $dbt = $this->tfc->openDBtool("circular_redirects_p"); + $sql = "SELECT * FROM `pages` WHERE `wiki`='{$wiki}' AND `simple_case_checked`=0 LIMIT 1" ; + $result = $this->tfc->getSQL ( $dbt , $sql ) ; + $ret = false ; + if($o = $result->fetch_object()){ + $redirects = [] ; + $page_safe = $dbt->real_escape_string ( $o->page ) ; + $sql = "SELECT DISTINCT p2.page_title FROM page p1,pagelinks,page p2,redirect + WHERE p1.page_id=pl_from AND pl_namespace=0 AND pl_from_namespace=0 AND p1.page_title='{$page_safe}' + AND p2.page_namespace=0 AND p2.page_title=pl_title AND rd_from=p2.page_id AND rd_namespace=0 AND rd_title=p1.page_title" ; + $dbw = $this->tfc->openDBwiki($wiki); + $result = $this->tfc->getSQL ( $dbw , $sql ) ; + while($o2 = $result->fetch_object()) $redirects[] = $o2->page_title; + $ret = $this->simple_fix($wiki,$o->page,$redirects); + if ( $ret ) { + $server = $this->tfc->getWebserverForWiki($wiki); + print "https://{$server}/wiki/".urlencode($o->page)."\n"; + } + $sql = "UPDATE `pages` SET `simple_case_checked`=1 WHERE `wiki`='{$wiki}' AND `page`='{$page_safe}'" ; + $this->tfc->getSQL ( $dbt , $sql ) ; + } else { + throw new Exception("No more candidates") ; + } + return $ret ; + } + + public function simple_cases($wiki) { + while ( true ) { + try { + $this->fix_next_page($wiki); + } catch (Exception $ex) { + return ; + } + } + } +} + +$cr = new CircularRedirects ; +if ( $argv[1] == 'update' ) $cr->update_db($argv[2]); +else if ( $argv[1] == 'simple' ) $cr->simple_cases($argv[2]); + +?> \ No newline at end of file From 68c34ee04d3c6d68fb9e2106821f9686a96d2075 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Wed, 30 Nov 2022 11:18:29 +0000 Subject: [PATCH 26/45] bot v1 --- scripts/circular_redirects/circular_redirects.php | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/circular_redirects/circular_redirects.php b/scripts/circular_redirects/circular_redirects.php index 206458e..c399607 100755 --- a/scripts/circular_redirects/circular_redirects.php +++ b/scripts/circular_redirects/circular_redirects.php @@ -75,6 +75,7 @@ public function simple_fix($wiki,$page,$redirects) { if ( count($redirects) == 0 ) return ; $page_nice = ucfirst(preg_replace('/_/',' ',$page)); $wt = $this->tfc->getWikiPageText($wiki,$page); + if ( preg_match('/\{\{\s*[Bb]ots\s*\}\}/',$wt) ) return false ; # {{bots}} $wt_orig = $wt ; foreach ( $redirects AS $rtitle ) { if ( preg_match("/^[A-Za-z][a-z]+:/",$rtitle) ) continue ; # TODO skipping redirects with namespaces From 21589d8d6e5a2114e0bca11322bbf18b896339df Mon Sep 17 00:00:00 2001 From: Bryan Davis Date: Wed, 21 Dec 2022 15:58:44 -0700 Subject: [PATCH 27/45] toolinfo: Update tool URLs * http -> https * tools.wmflabs.org/* -> *.toolforge.org --- public_html/toolinfo.json | 176 +++++++++++++++++++------------------- 1 file changed, 88 insertions(+), 88 deletions(-) diff --git a/public_html/toolinfo.json b/public_html/toolinfo.json index 25c83d5..8190f1f 100644 --- a/public_html/toolinfo.json +++ b/public_html/toolinfo.json @@ -3,7 +3,7 @@ "name" : "mm_wdq" , "title" : "WikiData Query (WDQ)", "description" : "A powerful API to interrogate Wikidata, used by many tools and third-party projects.", - "url" : "http://wdq.wmflabs.org", + "url" : "https://wdq.wmflabs.org", "keywords" : "wikidata, query, wdq, pagepile", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/wikidataquery" @@ -12,7 +12,7 @@ "name" : "mm_wikidata_game" , "title" : "The Wikidata Game", "description" : "A set of games to quickly add statements to Wikidata.", - "url" : "http://tools.wmflabs.org/wikidata-game", + "url" : "https://wikidata-game.toolforge.org", "keywords" : "wikidata, game, gamification, api", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/wikidata-game" @@ -21,7 +21,7 @@ "name" : "mm_terminator" , "title" : "Wikidata Terminator", "description" : "Shows identical terms (labels) with missing descriptions, popular items with missing labels etc.", - "url" : "http://tools.wmflabs.org/wikidata-terminator", + "url" : "https://wikidata-terminator.toolforge.org", "keywords" : "wikidata, label, language, translation, pagepile", "author" : "Magnus Manske" }, @@ -29,7 +29,7 @@ "name" : "mm_reasonator" , "title" : "Reasonator", "description" : "Human-readable display of Wikidata.", - "url" : "http://tools.wmflabs.org/reasonator", + "url" : "https://reasonator.toolforge.org", "keywords" : "wikidata, reasonator, api, visual, language, sparql", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/reasonator" @@ -38,7 +38,7 @@ "name" : "mm_nitol", "title" : "Not in the other language", "description" : "Finds Wikidata items with a Wikipedia page in language A but not in B.", - "url" : "http://tools.wmflabs.org/not-in-the-other-language", + "url" : "https://not-in-the-other-language.toolforge.org", "keywords" : "wikidata, interwiki, language, pagepile", "author" : "Magnus Manske" }, @@ -46,7 +46,7 @@ "name" : "mm_mixnmatch", "title" : "Mix'n'match", "description" : "GLAM catalog/Wikidata matching tool; red link lists on steroids.", - "url" : "http://tools.wmflabs.org/mix-n-match", + "url" : "https://mix-n-match.toolforge.org", "keywords" : "glam, wikidata, redlinks", "author" : "Magnus Manske" }, @@ -55,7 +55,7 @@ "name": "mm_glamorous2", "title": "GLAMorous 2", "description": "A tool to keep track usage and views of Commons images on other projects.", - "url": "http://tools.wmflabs.org/glamtools/glamorous/", + "url": "https://glamtools.toolforge.org/glamorous/", "keywords": "glam, files, images, views, commons", "author": "Magnus Manske", "repository": "https://bitbucket.org/magnusmanske/glamtools" @@ -65,7 +65,7 @@ "name" : "mm_flickr2commons", "title" : "Flickr2Commons", "description" : "Can upload single and multiple files from Flickr to Commons.", - "url" : "http://tools.wmflabs.org/flickr2commons", + "url" : "https://flickr2commons.toolforge.org", "keywords" : "commons, flickr, files, multimedia, transfer, copy, tooltranslate", "author" : "Magnus Manske" }, @@ -73,7 +73,7 @@ "name" : "mm_geograph2commons", "title" : "Geograph2Commons", "description" : "Can upload a single file from geograph.co.uk etc. to Commons.", - "url" : "http://tools.wmflabs.org/geograph2commons", + "url" : "https://geograph2commons.toolforge.org", "keywords" : "commons, file, transfer, copy, geograph", "author" : "Magnus Manske" }, @@ -81,7 +81,7 @@ "name" : "mm_commonshelper", "title" : "CommonsHelper", "description" : "Can transfer files from wiki(p|m)edia to Commons.", - "url" : "http://tools.wmflabs.org/commonshelper", + "url" : "https://commonshelper.toolforge.org", "keywords" : "commons, wikipedia, files, transfer, copy", "author" : "Magnus Manske" }, @@ -89,7 +89,7 @@ "name" : "mm_add_information", "title" : "Add information", "description" : "Adds an {{Information}} template to a file on Commons.", - "url" : "http://tools.wmflabs.org/add-information", + "url" : "https://add-information.toolforge.org", "keywords" : "commons, information, template", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/add_information" @@ -98,7 +98,7 @@ "name" : "mm_glamorous", "title" : "GLAMorous", "description" : "A tool to keep track of Commons images used on other projects.", - "url" : "http://tools.wmflabs.org/glamtools/glamorous.php", + "url" : "https://glamtools.toolforge.org/glamorous.php", "keywords" : "glam, files, images, views, commons", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/glamtools" @@ -107,7 +107,7 @@ "name" : "mm_baglama", "title" : "BaGLAMa", "description" : "Monthly view counts for pages containing images from selected Commons categories. Supersedec by BaGLAMa2.", - "url" : "http://tools.wmflabs.org/glamtools/baglama.php", + "url" : "https://glamtools.toolforge.org/baglama.php", "keywords" : "glam, files, images, views, commons", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/glamtools" @@ -116,7 +116,7 @@ "name" : "mm_baglama2", "title" : "BaGLAMa 2", "description" : "Monthly view counts for pages containing images from selected Commons categories.", - "url" : "http://tools.wmflabs.org/glamtools/baglama2/", + "url" : "https://glamtools.toolforge.org/baglama2/", "keywords" : "glam, files, images, views, commons", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/glamtools" @@ -125,7 +125,7 @@ "name" : "mm_treeviews", "title" : "TreeViews", "description" : "Monthly view statistics for category trees.", - "url" : "http://tools.wmflabs.org/glamtools/treeviews/", + "url" : "https://glamtools.toolforge.org/treeviews/", "keywords" : "glam, files, images, views, commons", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/glamtools" @@ -134,7 +134,7 @@ "name" : "mm_unused_files", "title" : "Unused files", "description" : "A list of files not used on Wikipedia, from a Commons category tree.", - "url" : "http://tools.wmflabs.org/magnustools/unused_files.php", + "url" : "https://magnustools.toolforge.org/unused_files.php", "keywords" : "glam, files, images, views, commons, list, unused", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -143,7 +143,7 @@ "name" : "mm_deep_insight", "title" : "Deep InSight", "description" : "Shows unsighted pages by category, including subcategories.", - "url" : "http://tools.wmflabs.org/sighting/deep_insight.php", + "url" : "https://sighting.toolforge.org/deep_insight.php", "keywords" : "list, wikipedia, flagged revisions", "author" : "Magnus Manske" } , @@ -151,7 +151,7 @@ "name" : "mm_deep_out_of_sight", "title" : "Deep out-of-sight", "description" : "Shows out-of-date sighted versions by category, including subcategories.", - "url" : "http://tools.wmflabs.org/sighting/deep_out_of_sight.php", + "url" : "https://sighting.toolforge.org/deep_out_of_sight.php", "keywords" : "list, wikipedia, flagged revisions", "author" : "Magnus Manske" } , @@ -159,7 +159,7 @@ "name" : "mm_random_out_of_sight", "title" : "Random out-of-sight", "description" : "Shows out-of-date sighted versions that only differ a little by size (typo fixes etc).", - "url" : "http://tools.wmflabs.org/sighting/random_out_of_sight.php", + "url" : "https://sighting.toolforge.org/random_out_of_sight.php", "keywords" : "list, wikipedia, flagged revisions", "author" : "Magnus Manske" } , @@ -167,7 +167,7 @@ "name" : "mm_out_of_sight", "title" : "Out-of-sight", "description" : "Randomly shows articles in de.wikipedia with out-of-date sighted versions.", - "url" : "http://tools.wmflabs.org/sighting/outofsight.php", + "url" : "https://sighting.toolforge.org/outofsight.php", "keywords" : "list, wikipedia, flagged revisions", "author" : "Magnus Manske" } , @@ -175,7 +175,7 @@ "name" : "mm_book2scroll", "title" : "Book2scroll", "description" : "A book reader for Wikisource.", - "url" : "http://tools.wmflabs.org/book2scroll", + "url" : "https://book2scroll.toolforge.org", "keywords" : "wikisource, book, reader", "author" : "Magnus Manske" } , @@ -183,7 +183,7 @@ "name" : "mm_map2wp", "title" : "Map2WP", "description" : "Attempts to map DNB articles on wikisource to wikipedia.", - "url" : "http://tools.wmflabs.org/dnbtools/map2wp.php", + "url" : "https://dnbtools.toolforge.org/map2wp.php", "keywords" : "wikisource, DNB, wikipedia", "author" : "Magnus Manske" } , @@ -191,7 +191,7 @@ "name" : "mm_wikisource_dnb", "title" : "Wikisource DNB", "description" : "Looks for inconsistencies in the wikisource link structures of the DNB.", - "url" : "http://tools.wmflabs.org/dnbtools/dnb_wikisource.php", + "url" : "https://dnbtools.toolforge.org/dnb_wikisource.php", "keywords" : "wikisource, dnb", "author" : "Magnus Manske" } , @@ -199,7 +199,7 @@ "name" : "mm_dnb_ratios", "title" : "DNB ratios", "description" : "Gives page length ratios between Wikisource and Wikipedia articles.", - "url" : "http://tools.wmflabs.org/dnbtools/dnb_ratios.php", + "url" : "https://dnbtools.toolforge.org/dnb_ratios.php", "keywords" : "wikisource, dnb, wikipedia", "author" : "Magnus Manske" } , @@ -207,7 +207,7 @@ "name" : "mm_dnb_link_finder", "title" : "DNB link finder", "description" : "Finds links to Google books DNB that would better point to Wikisource.", - "url" : "http://tools.wmflabs.org/dnbtools/dnb_link_finder.php", + "url" : "https://dnbtools.toolforge.org/dnb_link_finder.php", "keywords" : "wikisource, dnb, google books", "author" : "Magnus Manske" } , @@ -215,7 +215,7 @@ "name" : "mm_geohack", "title" : "Geohack", "description" : "Meta-map tool. Used on Wikipedia. [example: Berlin]", - "url" : "http://tools.wmflabs.org/geohack/geohack.php?pagename=Berlin¶ms=52_31_N_13_23_E_type:city(3292365)_region:DE", + "url" : "https://geohack.toolforge.org/geohack.php?pagename=Berlin¶ms=52_31_N_13_23_E_type:city(3292365)_region:DE", "keywords" : "map, wikipedia, external", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/geohack" @@ -224,7 +224,7 @@ "name" : "mm_cas", "title" : "CAS", "description" : "Shows links to CAS numbers for chemicals. [example: Trisodium phosphate]", - "url" : "http://tools.wmflabs.org/magnustools/cas.php?language=en&cas=7601-54-9&title=Trisodium%20phosphate", + "url" : "https://magnustools.toolforge.org/cas.php?language=en&cas=7601-54-9&title=Trisodium%20phosphate", "keywords" : "wikipedia, infobox, cas, chemicals, external", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -233,7 +233,7 @@ "name" : "mm_isin", "title" : "ISIN", "description" : "Shows links to ISIN numbers (stock market) based on parameter.", - "url" : "http://tools.wmflabs.org/isin/?language=de&isin=GB00B24CGK77", + "url" : "https://isin.toolforge.org/?language=de&isin=GB00B24CGK77", "keywords" : "wikipedia, infobox, isin, stock market, external", "author" : "Magnus Manske" } , @@ -241,7 +241,7 @@ "name" : "mm_prep_bio", "title" : "PrepBio", "description" : "Helps to generate infobox templates for biographies.", - "url" : "http://tools.wmflabs.org/magnustools/prepbio.php", + "url" : "https://magnustools.toolforge.org/prepbio.php", "keywords" : "wikipedia, wikitext, generator, infobox", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -250,7 +250,7 @@ "name" : "mm_html2wiki", "title" : "HTML2wiki", "description" : "Can convert some HTML into wikitext, especially tables.", - "url" : "http://tools.wmflabs.org/magnustools/html2wiki.php", + "url" : "https://magnustools.toolforge.org/html2wiki.php", "keywords" : "wikipedia, wikitext, generator, html, tables", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -259,7 +259,7 @@ "name" : "mm_tab2wiki", "title" : "Tab2Wiki", "description" : "Converts tables (tab-delimited, e.g. copied from Excel) to Wikitext tables.", - "url" : "http://tools.wmflabs.org/magnustools/tab2wiki.php", + "url" : "https://magnustools.toolforge.org/tab2wiki.php", "keywords" : "wikipedia, wikitext, generator, tables", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -268,7 +268,7 @@ "name" : "mm_wd_reconcile", "title" : "Wikidata Reconcile", "description" : "A reconciliation API for Wikidata.", - "url" : "https://tools.wmflabs.org/wikidata-reconcile/", + "url" : "https://wikidata-reconcile.toolforge.org/", "keywords" : "wikidata, reconcile, reconciliation, api", "author" : "Magnus Manske" } , @@ -276,7 +276,7 @@ "name" : "mm_pagepile", "title" : "PagePile", "description" : "A tool to generate and manage lists of pages, and to use them across many other tools.", - "url" : "https://tools.wmflabs.org/pagepile/", + "url" : "https://pagepile.toolforge.org/", "keywords" : "list, lists, wikipedia, wikidata, generator, pagepile", "author" : "Magnus Manske" } , @@ -284,7 +284,7 @@ "name" : "mm_pagepile2kml", "title" : "PagePile2KML", "description" : "Generates KML from a PagePile list of Wikidata items or articles (via Wikidata).", - "url" : "https://tools.wmflabs.org/wikidata-todo/pagepile2kml.php", + "url" : "https://wikidata-todo.toolforge.org/pagepile2kml.php", "keywords" : "KML, map, maps, wikipedia, wikidata, pagepile", "author" : "Magnus Manske" } , @@ -292,7 +292,7 @@ "name" : "mm_makeref", "title" : "MakeRef", "description" : "A form to generate references.", - "url" : "http://tools.wmflabs.org/makeref/", + "url" : "https://makeref.toolforge.org/", "keywords" : "wikipedia, wikitext, generator, references", "author" : "Magnus Manske" } , @@ -300,7 +300,7 @@ "name" : "mm_isbn2wiki", "title" : "ISBN2Wiki", "description" : "Generates wikitext reference texts for ISBNs.", - "url" : "http://tools.wmflabs.org/isbn2wiki/", + "url" : "https://isbn2wiki.toolforge.org/", "keywords" : "wikitext, wikipedia, generator, isbn, references", "author" : "Magnus Manske" } , @@ -308,7 +308,7 @@ "name" : "mm_get_article_intro", "title" : "GetArticleIntro", "description" : "Returns the first sentence of an article as JSONP.", - "url" : "http://tools.wmflabs.org/magnustools/get_article_intro.php", + "url" : "https://magnustools.toolforge.org/get_article_intro.php", "keywords" : "wikipedia, api, extraction, jsonp", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -317,7 +317,7 @@ "name" : "mm_update_article_counter", "title" : "Update article counter", "description" : "Updates the number of articles in a WikiProject.", - "url" : "http://tools.wmflabs.org/magnustools/update_article_counter.php", + "url" : "https://magnustools.toolforge.org/update_article_counter.php", "keywords" : "wikipedia, wikiproject, editing", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -326,7 +326,7 @@ "name" : "mm_templator", "title" : "Templator", "description" : "Generates templates from forms.", - "url" : "http://tools.wmflabs.org/templator/", + "url" : "https://templator.toolforge.org/", "keywords" : "wikitext, templates, generator", "author" : "Magnus Manske" } , @@ -334,7 +334,7 @@ "name" : "mm_missing_topics", "title" : "MissingTopics", "description" : "Finds the 'most wanted' missing topics in an article or category tree.", - "url" : "http://tools.wmflabs.org/missingtopics", + "url" : "https://missingtopics.toolforge.org", "keywords" : "missing, redlinks, wikipedia, list, category", "author" : "Magnus Manske" } , @@ -342,7 +342,7 @@ "name" : "mm_wiki_todo", "title" : "Wiki ToDo", "description" : "Finds articles with things to do for you!", - "url" : "http://tools.wmflabs.org/wiki-todo/", + "url" : "https://wiki-todo.toolforge.org/", "keywords" : "wikipedia, todo", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/wikidata-todo" @@ -351,7 +351,7 @@ "name" : "mm_most_wanted", "title" : "Most wanted", "description" : "Topics, measured by redlink count, for several languages.", - "url" : "http://tools.wmflabs.org/most-wanted/", + "url" : "https://most-wanted.toolforge.org/", "keywords" : "missing, redlinks, list", "author" : "Magnus Manske" } , @@ -359,7 +359,7 @@ "name" : "mm_get_distinct_authors", "title" : "GetDistinctAuthors", "description" : "Gives the distinct (non-IP) list of editors on a set of articles.", - "url" : "http://tools.wmflabs.org/magnustools/get_distinct_authors.php", + "url" : "https://magnustools.toolforge.org/get_distinct_authors.php", "keywords" : "authors, editors, users, wikipedia, list", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -368,7 +368,7 @@ "name" : "mm_herding_sheep", "title" : "Herding Sheep", "description" : "Shows the (last) edits of all users in a user category.", - "url" : "http://tools.wmflabs.org/magnustools/herding_sheep.php", + "url" : "https://magnustools.toolforge.org/herding_sheep.php", "keywords" : "authors, editors, users, wikipedia, list", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -377,7 +377,7 @@ "name" : "mm_trans_parent", "title" : "Trans-parent", "description" : "Finds potential translators between two languages amongst Wikipedians.", - "url" : "http://tools.wmflabs.org/magnustools/transparent.php", + "url" : "https://magnustools.toolforge.org/transparent.php", "keywords" : "translation, wikipedia, editors, users, authors", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -386,7 +386,7 @@ "name" : "mm_data_url", "title" : "Data URL", "description" : "Demo for images as data URLs in Wikipedia, with or without WEBP.", - "url" : "http://tools.wmflabs.org/magnustools/wp_data_url.php?lang=en&title=Richard_Dawkins", + "url" : "https://magnustools.toolforge.org/wp_data_url.php?lang=en&title=Richard_Dawkins", "keywords" : "html, demo, images, webp, wikipedia", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -395,7 +395,7 @@ "name" : "mm_joanjoc", "title" : "JoanJoc's tools", "description" : "Something with language links.", - "url" : "http://tools.wmflabs.org/joanjoc/", + "url" : "https://joanjoc.toolforge.org/", "keywords" : "interlanguage", "author" : "Magnus Manske, JoanJoc" } , @@ -403,7 +403,7 @@ "name" : "mm_no_information", "title" : "No Information", "description" : "Files with no {{Information}} template. Merged from 'all_no_info' and 'mynoinfo' on toolserver.", - "url" : "http://tools.wmflabs.org/add-information/no_information.php", + "url" : "https://add-information.toolforge.org/no_information.php", "keywords" : "commons, files, information, missing", "author" : "Magnus Manske" } , @@ -411,7 +411,7 @@ "name" : "mm_usual_suspects", "title" : "The Usual Suspects", "description" : "File management tool, merged 'Bad Boys' and 'Bad Old Ones' from toolserver.", - "url" : "http://tools.wmflabs.org/usualsuspects/", + "url" : "https://usualsuspects.toolforge.org/", "keywords" : "commons, files", "author" : "Magnus Manske" } , @@ -419,7 +419,7 @@ "name" : "mm_file_reuse", "title" : "File reuse", "description" : "Allows you to drag'n'drop a file onto it, and show a thumbnail and attribution.", - "url" : "http://tools.wmflabs.org/magnustools/file_reuse/", + "url" : "https://magnustools.toolforge.org/file_reuse/", "keywords" : "commons, files, reuse, attribution, demo, drag'n'drop", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -428,7 +428,7 @@ "name" : "mm_fist", "title" : "FIST", "description" : "The Free Image Search Tool.", - "url" : "http://tools.wmflabs.org/fist/fist.php", + "url" : "https://fist.toolforge.org/fist.php", "keywords" : "external, search, images, files, commons, wikipedia, flickr, geograph, pagepile", "author" : "Magnus Manske" } , @@ -436,7 +436,7 @@ "name" : "mm_url2commons", "title" : "URL2Commons", "description" : "Can transfer files from a list of URLs to Commons.", - "url" : "http://tools.wmflabs.org/url2commons/index.html", + "url" : "https://url2commons.toolforge.org/index.html", "keywords" : "commons, files, images, upload, external", "author" : "Magnus Manske" } , @@ -444,7 +444,7 @@ "name" : "mm_file_siblings", "title" : "File siblings", "description" : "Builds, for a file, an HTML fragment with thumbnails of related images.", - "url" : "http://tools.wmflabs.org/file-siblings/", + "url" : "https://file-siblings.toolforge.org/", "keywords" : "commons, files, category, gallery, wikipedia", "author" : "Magnus Manske" } , @@ -452,7 +452,7 @@ "name" : "mm_file_siblings_interface", "title" : "File siblings interface", "description" : "Wikipedia JavaScript interface for File siblings.", - "url" : "http://de.wikipedia.org/wiki/MediaWiki:Gadget-ImageSiblings.js", + "url" : "https://de.wikipedia.org/wiki/MediaWiki:Gadget-ImageSiblings.js", "keywords" : "commons, files, category, gallery, wikipedia, interface, javascript", "author" : "Magnus Manske" } , @@ -460,7 +460,7 @@ "name" : "mm_file_dupes", "title" : "FileDupes", "description" : "For finding image duplicates. A merge of 'userdupes' and 'commons_dupes' from toolserver.", - "url" : "http://tools.wmflabs.org/filedupes/", + "url" : "https://filedupes.toolforge.org/", "keywords" : "commons, wikipedia, images, files, duplicates", "author" : "Magnus Manske" } , @@ -468,7 +468,7 @@ "name" : "mm_multidesc", "title" : "MultiDesc", "description" : "Generates multilingual gallery and file descriptions (a merge of 'whatisthat' and 'commons_sumitup' from toolserver).", - "url" : "http://tools.wmflabs.org/multidesc/", + "url" : "https://multidesc.toolforge.org/", "keywords" : "commons, images, files, descriptions, generator", "author" : "Magnus Manske" } , @@ -476,7 +476,7 @@ "name" : "mm_wiki2playlist", "title" : "Wiki2playlist", "description" : "Generates a .M3U playlist from files hosted on Commons.", - "url" : "http://tools.wmflabs.org/magnustools/wiki2playlist.php", + "url" : "https://magnustools.toolforge.org/wiki2playlist.php", "keywords" : "commons, multimedia, audio, m3u, playlist", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -485,7 +485,7 @@ "name" : "mm_icommons", "title" : "iCommons", "description" : "A Commons browser for mobile devices.", - "url" : "http://tools.wmflabs.org/icommons/", + "url" : "https://icommons.toolforge.org/", "keywords" : "commons, mobile, visualisation", "author" : "Magnus Manske" } , @@ -493,7 +493,7 @@ "name" : "mm_flickrfree", "title" : "FlickrFree", "description" : "Shows free, recently uploaded images on Flickr, ready for transfer to Commons.", - "url" : "http://tools.wmflabs.org/magnustools/flickrfree.php", + "url" : "https://magnustools.toolforge.org/flickrfree.php", "keywords" : "flickr, images, external", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -502,7 +502,7 @@ "name" : "mm_catscan2", "title" : "CatScan 2", "description" : "For complex category tree and template intersection.", - "url" : "http://tools.wmflabs.org/catscan2/catscan2.php", + "url" : "https://catscan2.toolforge.org/catscan2.php", "keywords" : "category, categories, templates, wikipedia, trees, query, pagepile", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/catscan2" @@ -511,7 +511,7 @@ "name" : "mm_reverse_category_tree", "title" : "Reverse category tree", "description" : "Get the categories a page or file is in, all the way down to the roots.", - "url" : "http://tools.wmflabs.org/catscan2/reverse_tree.php", + "url" : "https://catscan2.toolforge.org/reverse_tree.php", "keywords" : "category, categories, wikipedia, commons, query, trees", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/catscan2" @@ -520,7 +520,7 @@ "name" : "mm_pages_in_categories", "title" : "Pages in categories", "description" : "Counts pages in a category tree.", - "url" : "http://tools.wmflabs.org/catscan2/pages_in_cats.php", + "url" : "https://catscan2.toolforge.org/pages_in_cats.php", "keywords" : "wikipedia, category, categories", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/catscan2" @@ -529,7 +529,7 @@ "name" : "mm_catfood", "title" : "CatFood", "description" : "RSS feeds for pages and files based on category or user name.", - "url" : "http://tools.wmflabs.org/catfood/catfood.php", + "url" : "https://catfood.toolforge.org/catfood.php", "keywords" : "category, categories, rss, files, images", "author" : "Magnus Manske" } , @@ -537,7 +537,7 @@ "name" : "mm_commons_image_feed", "title" : "Commons image feed", "description" : "RSS image feeds for Commons categories to use in screensavers etc.", - "url" : "http://tools.wmflabs.org/catfood/commons_image_feed.php", + "url" : "https://catfood.toolforge.org/commons_image_feed.php", "keywords" : "commons, rss, screensaver, images, categories", "author" : "Magnus Manske" } , @@ -545,7 +545,7 @@ "name" : "mm_catnap", "title" : "CatNap", "description" : "Lists Wikipedia articles in a category grouped by their other categories.", - "url" : "http://tools.wmflabs.org/catnap", + "url" : "https://catnap.toolforge.org", "keywords" : "wikipedia, categories, category, tree, query", "author" : "Magnus Manske" } , @@ -553,7 +553,7 @@ "name" : "mm_crosscats", "title" : "CrossCats", "description" : "Finds articles that ought to be in a category tree in a language via categories in other languages.", - "url" : "http://tools.wmflabs.org/catscan2/cross_cats.php", + "url" : "https://catscan2.toolforge.org/cross_cats.php", "keywords" : "category, categories, wikipedia, interlanguage, tree, query", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/catscan2" @@ -562,7 +562,7 @@ "name" : "mm_random_page_in_cat", "title" : "Random page in category", "description" : "Redirects to a random page in a category.", - "url" : "http://tools.wmflabs.org/magnustools/randomarticle.php", + "url" : "https://magnustools.toolforge.org/randomarticle.php", "keywords" : "random, wikipedia, category, categories, tree, redirect", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/magnustools" @@ -571,7 +571,7 @@ "name" : "mm_toolscript", "title" : "ToolScript", "description" : "An interface to write and run JavaScript code, with convenient classes to get category trees, run WDQ queries etc.", - "url" : "https://tools.wmflabs.org/toolscript/index.html", + "url" : "https://toolscript.toolforge.org/index.html", "keywords" : "javascript, scripting, code, wdq, categories, tree", "author" : "Magnus Manske", "repository" : "https://bitbucket.org/magnusmanske/toolscript" @@ -580,7 +580,7 @@ "name" : "mm_misstake", "title" : "Ranked missing articles", "description" : "Finds and ranks missing articles from page links", - "url" : "http://tools.wmflabs.org/magnus-toolserver/misstake.php", + "url" : "https://magnus-toolserver.toolforge.org/misstake.php", "keywords" : "wikipedia, list, missing", "author" : "Magnus Manske" } , @@ -588,7 +588,7 @@ "name" : "mm_redirector", "title" : "Redirector", "description" : "Finds wanted redirects across WIkipedia languages", - "url" : "http://tools.wmflabs.org/magnus-toolserver/redirector.php", + "url" : "https://magnus-toolserver.toolforge.org/redirector.php", "keywords" : "wikipedia, redirect, redirects, languages", "author" : "Magnus Manske" } , @@ -596,7 +596,7 @@ "name" : "mm_persondata", "title" : "Persondata", "description" : "Generates {{Persondata}} template (de/en)", - "url" : "http://tools.wmflabs.org/magnus-toolserver/persondata.php", + "url" : "https://magnus-toolserver.toolforge.org/persondata.php", "keywords" : "wikipedia, wikitext, generator, persondata, biography", "author" : "Magnus Manske" } , @@ -604,7 +604,7 @@ "name" : "mm_related_places", "title" : "Related places", "description" : "Generates KML for all places related to (linked from) an article", - "url" : "http://tools.wmflabs.org/magnus-toolserver/related_places.php", + "url" : "https://magnus-toolserver.toolforge.org/related_places.php", "keywords" : "wikipedia, kml, maps", "author" : "Magnus Manske" } , @@ -612,7 +612,7 @@ "name" : "mm_bong", "title" : "Bong", "description" : "Bing-like Wikimedia search interface.", - "url" : "http://tools.wmflabs.org/magnus-toolserver/bong.php", + "url" : "https://magnus-toolserver.toolforge.org/bong.php", "keywords" : "", "author" : "Magnus Manske" } , @@ -620,7 +620,7 @@ "name" : "mm_commons_api", "title" : "Commons API", "description" : "An experimental API to return author and media data from Commons as XML.", - "url" : "http://tools.wmflabs.org/magnus-toolserver/commonsapi.php", + "url" : "https://magnus-toolserver.toolforge.org/commonsapi.php", "keywords" : "commons, files, api", "author" : "Magnus Manske" } , @@ -636,7 +636,7 @@ "name" : "mm_wd_species_commons", "title" : "Wikidata species images", "description" : "Wikidata items about species that have no image, but potential images on Commons.", - "url" : "http://tools.wmflabs.org/fist/wd_species_commons.php", + "url" : "https://fist.toolforge.org/wd_species_commons.php", "keywords" : "wikidata, commons, images, files", "author" : "Magnus Manske" } , @@ -644,7 +644,7 @@ "name" : "mm_edit_counter", "title" : "Quick edit counter", "description" : "Simple user edit count breakdown, as a replacement for when supercount is down.", - "url" : "http://tools.wmflabs.org/magnustools/quick_counter.php", + "url" : "https://magnustools.toolforge.org/quick_counter.php", "keywords" : "edit, editcount", "author" : "Magnus Manske" } , @@ -652,7 +652,7 @@ "name" : "mm_autodesc", "title" : "AutoDesc", "description" : "Wikidata automated description API", - "url" : "http://tools.wmflabs.org/autodesc", + "url" : "https://autodesc.toolforge.org", "keywords" : "wikidata, api, description", "author" : "Magnus Manske" } , @@ -660,7 +660,7 @@ "name" : "mm_get_item_names", "title" : "Get item names", "description" : "Generates a list of Wikidata labels for a list of items, with language fallback", - "url" : "https://tools.wmflabs.org/wikidata-todo/get_item_names.php", + "url" : "https://wikidata-todo.toolforge.org/get_item_names.php", "keywords" : "wikidata, label, labels, name, names, language, languages, pagepile", "author" : "Magnus Manske" } , @@ -668,7 +668,7 @@ "name" : "mm_linked_items", "title" : "Linked items", "description" : "Generates a list of Wikidata items, from wikitext links or PagePile", - "url" : "https://tools.wmflabs.org/wikidata-todo/get_item_names.php", + "url" : "https://wikidata-todo.toolforge.org/get_item_names.php", "keywords" : "wikidata, item, items, link, links, wikipedia, pagepile", "author" : "Magnus Manske" } , @@ -676,7 +676,7 @@ "name" : "mm_wikishootme" , "title" : "Wiki ShootMe!" , "description" : "Shows Wikidata items in your area that do not have an image" , - "url" : "https://tools.wmflabs.org/wikishootme/" , + "url" : "https://wikishootme.toolforge.org/" , "keywords" : "wikidata, commons, image, around, coordinate, coordinates" , "author" : "Magnus Manske" , "repository" : "" @@ -685,7 +685,7 @@ "name" : "mm_wikisoba" , "title" : "WikiSoba" , "description" : "A demo for a quiz system, can utilize Wikipedia pages as page source" , - "url" : "https://tools.wmflabs.org/wikisoba/" , + "url" : "https://wikisoba.toolforge.org/" , "keywords" : "quiz, demo" , "author" : "Magnus Manske" , "repository" : "https://bitbucket.org/magnusmanske/wikisoba" @@ -694,7 +694,7 @@ "name" : "mm_widar" , "title" : "WiDaR" , "description" : "OAuth login wrapper for several other tools. Wikidata-centric." , - "url" : "https://tools.wmflabs.org/widar/" , + "url" : "https://widar.toolforge.org/" , "keywords" : "wikidata, oauth, login" , "author" : "Magnus Manske" , "repository" : "" @@ -703,7 +703,7 @@ "name" : "mm_listeria" , "title" : "Listeria" , "description" : "Generates lists on Wikipedias, based on Wikidata queries. Daily updates." , - "url" : "https://tools.wmflabs.org/listeria/" , + "url" : "https://listeria.toolforge.org/" , "keywords" : "list, lists, wdq, sparql, wikidata, query, generator" , "author" : "Magnus Manske" , "repository" : "https://bitbucket.org/magnusmanske/listeria" @@ -712,7 +712,7 @@ "name" : "mm_metamine" , "title" : "MetaMine" , "description" : "Demo tool to show crossover between Wikidata and ContentMine" , - "url" : "https://tools.wmflabs.org/metamine/" , + "url" : "https://metamine.toolforge.org/" , "keywords" : "wikidata, contentmine, demo" , "author" : "Magnus Manske" , "repository" : "" @@ -721,7 +721,7 @@ "name" : "mm_tooldir" , "title" : "Tool directory" , "description" : "Alternative display of Hay's tool directory" , - "url" : "https://tools.wmflabs.org/magnustools/hay2.html" , + "url" : "https://magnustools.toolforge.org/hay2.html" , "keywords" : "tools, directory, demo" , "author" : "Magnus Manske" , "repository" : "" @@ -739,7 +739,7 @@ "name" : "mm_tooltranslate" , "title" : "Tool Translate" , "description" : "Community-driven translation of tool interfaces" , - "url" : "https://tools.wmflabs.org/tooltranslate/" , + "url" : "https://tooltranslate.toolforge.org/" , "keywords" : "translation, interface, i18n, tooltranslate" , "author" : "Magnus Manske" , "repository" : "https://bitbucket.org/magnusmanske/tooltranslate" @@ -748,8 +748,8 @@ "name" : "mm_wdfist", "title" : "Wikidata FIST", "description" : "The Free Image Search Tool for Wikidata. Find items without image, but images in associated Wikipedia articles.", - "url" : "http://tools.wmflabs.org/fist/wdfist/index.html", + "url" : "https://fist.toolforge.org/wdfist/index.html", "keywords" : "wikidata, images, files, wdq, pagepile, sparql, petscan, tooltranslate", "author" : "Magnus Manske" } -] \ No newline at end of file +] From 5982cc564f7c769780f789af0814cafd248554ad Mon Sep 17 00:00:00 2001 From: Bryan Davis Date: Wed, 21 Dec 2022 16:10:02 -0700 Subject: [PATCH 28/45] toolinfo: update WikiShootMe record * Change title to match on-wiki documentation * Update description to match lead sentence from wiki * Add missing repository URL --- public_html/toolinfo.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public_html/toolinfo.json b/public_html/toolinfo.json index 8190f1f..5bd03fa 100644 --- a/public_html/toolinfo.json +++ b/public_html/toolinfo.json @@ -674,12 +674,12 @@ } , { "name" : "mm_wikishootme" , - "title" : "Wiki ShootMe!" , - "description" : "Shows Wikidata items in your area that do not have an image" , + "title" : "WikiShootMe" , + "description" : "WikiShootMe V3 (WSM) is a tool to show Wikidata items, Wikipedia articles, and Commons images with coordinates, all on the same map." , "url" : "https://wikishootme.toolforge.org/" , "keywords" : "wikidata, commons, image, around, coordinate, coordinates" , "author" : "Magnus Manske" , - "repository" : "" + "repository" : "https://bitbucket.org/magnusmanske/wikishootme" } , { "name" : "mm_wikisoba" , From cda76a05bfa177c8e6fd6c0831503e18201f6b50 Mon Sep 17 00:00:00 2001 From: Bryan Davis Date: Wed, 21 Dec 2022 16:12:19 -0700 Subject: [PATCH 29/45] toolinfo: Mark WDQ as deprecated and replaced Add the newer toolinfo v1.2 depreciated and replaced_by attributes to inform folks that https://query.wikidata.org/ is the currently preferred solution. --- public_html/toolinfo.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/public_html/toolinfo.json b/public_html/toolinfo.json index 5bd03fa..14a907b 100644 --- a/public_html/toolinfo.json +++ b/public_html/toolinfo.json @@ -6,7 +6,9 @@ "url" : "https://wdq.wmflabs.org", "keywords" : "wikidata, query, wdq, pagepile", "author" : "Magnus Manske", - "repository" : "https://bitbucket.org/magnusmanske/wikidataquery" + "repository" : "https://bitbucket.org/magnusmanske/wikidataquery", + "deprecated" : true, + "replaced_by" : "https://query.wikidata.org/" }, { "name" : "mm_wikidata_game" , From b883716b447bb3f165882b1360b7fc08d2026fd8 Mon Sep 17 00:00:00 2001 From: Bryan Davis Date: Wed, 21 Dec 2022 16:16:32 -0700 Subject: [PATCH 30/45] toolinfo: Mark BaGLAMa as deprecated and replaced --- public_html/toolinfo.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/public_html/toolinfo.json b/public_html/toolinfo.json index 14a907b..ba23222 100644 --- a/public_html/toolinfo.json +++ b/public_html/toolinfo.json @@ -108,11 +108,13 @@ { "name" : "mm_baglama", "title" : "BaGLAMa", - "description" : "Monthly view counts for pages containing images from selected Commons categories. Supersedec by BaGLAMa2.", + "description" : "Monthly view counts for pages containing images from selected Commons categories. Superseded by BaGLAMa2.", "url" : "https://glamtools.toolforge.org/baglama.php", "keywords" : "glam, files, images, views, commons", "author" : "Magnus Manske", - "repository" : "https://bitbucket.org/magnusmanske/glamtools" + "repository" : "https://bitbucket.org/magnusmanske/glamtools", + "deprecated" : true, + "replaced_by" : "https://glamtools.toolforge.org/baglama2/" } , { "name" : "mm_baglama2", From 7b01b086beb0f1061b22e7932e17209bf8eafe70 Mon Sep 17 00:00:00 2001 From: Bryan Davis Date: Wed, 21 Dec 2022 16:29:10 -0700 Subject: [PATCH 31/45] toolinfo: Mark WiDaR as deprecated --- public_html/toolinfo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public_html/toolinfo.json b/public_html/toolinfo.json index ba23222..afa2d15 100644 --- a/public_html/toolinfo.json +++ b/public_html/toolinfo.json @@ -701,7 +701,7 @@ "url" : "https://widar.toolforge.org/" , "keywords" : "wikidata, oauth, login" , "author" : "Magnus Manske" , - "repository" : "" + "deprecated" : true } , { "name" : "mm_listeria" , From 6a5d2133ae68432285380abcee4897fbc2dd9aeb Mon Sep 17 00:00:00 2001 From: Bryan Davis Date: Wed, 21 Dec 2022 16:45:50 -0700 Subject: [PATCH 32/45] toolinfo: PetScan updates * Update description using text from metawiki * Update repository * Mark CatScan family of tools as deprecated and replaced by PetScan --- public_html/toolinfo.json | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/public_html/toolinfo.json b/public_html/toolinfo.json index afa2d15..0567088 100644 --- a/public_html/toolinfo.json +++ b/public_html/toolinfo.json @@ -509,7 +509,9 @@ "url" : "https://catscan2.toolforge.org/catscan2.php", "keywords" : "category, categories, templates, wikipedia, trees, query, pagepile", "author" : "Magnus Manske", - "repository" : "https://bitbucket.org/magnusmanske/catscan2" + "repository" : "https://bitbucket.org/magnusmanske/catscan2", + "deprecated" : true, + "replaced_by" : "https://petscan.wmflabs.org/" } , { "name" : "mm_reverse_category_tree", @@ -518,7 +520,9 @@ "url" : "https://catscan2.toolforge.org/reverse_tree.php", "keywords" : "category, categories, wikipedia, commons, query, trees", "author" : "Magnus Manske", - "repository" : "https://bitbucket.org/magnusmanske/catscan2" + "repository" : "https://bitbucket.org/magnusmanske/catscan2", + "deprecated" : true, + "replaced_by" : "https://petscan.wmflabs.org/" } , { "name" : "mm_pages_in_categories", @@ -527,7 +531,9 @@ "url" : "https://catscan2.toolforge.org/pages_in_cats.php", "keywords" : "wikipedia, category, categories", "author" : "Magnus Manske", - "repository" : "https://bitbucket.org/magnusmanske/catscan2" + "repository" : "https://bitbucket.org/magnusmanske/catscan2", + "deprecated" : true, + "replaced_by" : "https://petscan.wmflabs.org/" } , { "name" : "mm_catfood", @@ -560,7 +566,9 @@ "url" : "https://catscan2.toolforge.org/cross_cats.php", "keywords" : "category, categories, wikipedia, interlanguage, tree, query", "author" : "Magnus Manske", - "repository" : "https://bitbucket.org/magnusmanske/catscan2" + "repository" : "https://bitbucket.org/magnusmanske/catscan2", + "deprecated" : true, + "replaced_by" : "https://petscan.wmflabs.org/" } , { "name" : "mm_random_page_in_cat", @@ -733,11 +741,11 @@ { "name" : "mm_petscan" , "title" : "PetScan" , - "description" : "Replacement for CatScan2, QuickIntersection, Creator, Autolist etc." , + "description" : "PetScan can generate lists of Wikipedia (and related projects) pages or Wikidata items that match certain criteria, such as all pages in a certain category, or all items with a certain property. PetScan can also combine some temporary lists (here called 'sources') in various ways, to create a new one." , "url" : "https://petscan.wmflabs.org" , "keywords" : "categories, intersection, WDQS, SPARQL, pagepile, wikidata" , "author" : "Magnus Manske" , - "repository" : "https://bitbucket.org/magnusmanske/petscan" + "repository" : "https://github.com/magnusmanske/petscan_rs" } , { "name" : "mm_tooltranslate" , From e89cbe0ec307df1ebbdf4b16c047b3c71d32c847 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Tue, 21 Feb 2023 13:38:47 +0000 Subject: [PATCH 33/45] misc --- public_html/php/ToolforgeCommon.php | 14 ++ .../circular_redirects/circular_redirects.php | 7 +- scripts/monitor/monitor.php | 209 ++++++++++++++++++ tjs | 2 +- 4 files changed, 230 insertions(+), 2 deletions(-) create mode 100755 scripts/monitor/monitor.php diff --git a/public_html/php/ToolforgeCommon.php b/public_html/php/ToolforgeCommon.php index 4e6b9ec..04b5fd5 100644 --- a/public_html/php/ToolforgeCommon.php +++ b/public_html/php/ToolforgeCommon.php @@ -167,6 +167,20 @@ private function getDBpassword () /*:string*/ { } } + # Requires a replica.trove.my.cnf file with user, password, and server + public function openDBtrove ( $dbname = '' ) { + if ( isset ( $this->tool_user_name ) and $this->tool_user_name != '' ) $user = $this->tool_user_name ; + else $user = str_replace ( 'tools.' , '' , get_current_user() ) ; + $passwordfile = "/data/project/${user}/replica.trove.my.cnf" ; + $config = parse_ini_file( $passwordfile ); + $user = $config['user']; + $password = $config['password']; + $server = $config['host']; + $db = @new mysqli($server, $user, $password , $dbname); + assert ( $db->connect_errno == 0 , 'Unable to connect to database [' . $db->connect_error . ']' ) ; + return $db ; + } + public function openDBtool ( $dbname = '' , $server = '' , $force_user = '' , $persistent = false ) { $this->getDBpassword() ; if ( $dbname == '' ) $dbname = '_main' ; diff --git a/scripts/circular_redirects/circular_redirects.php b/scripts/circular_redirects/circular_redirects.php index c399607..b7752be 100755 --- a/scripts/circular_redirects/circular_redirects.php +++ b/scripts/circular_redirects/circular_redirects.php @@ -93,8 +93,12 @@ public function simple_fix($wiki,$page,$redirects) { } } if ( $wt!=$wt_orig ) { - $this->setPageText($wiki,$page,$wt,"Replacing circular redirect link (redirect links back to this page) with plain text"); $ret = true ; + try { + $this->setPageText($wiki,$page,$wt,"Replacing circular redirect link (redirect links back to this page) with plain text"); + } catch (Exception $ex) { + $ret = false ; # Some error + } } else { $ret = false ; } @@ -176,6 +180,7 @@ public function simple_cases($wiki) { try { $this->fix_next_page($wiki); } catch (Exception $ex) { + print_r($ex); return ; } } diff --git a/scripts/monitor/monitor.php b/scripts/monitor/monitor.php new file mode 100755 index 0000000..c6a69c5 --- /dev/null +++ b/scripts/monitor/monitor.php @@ -0,0 +1,209 @@ +#!/usr/bin/env php +tfc = new ToolforgeCommon(); + $this->testing = $testing; + } + + public function run() { + $this->check_quickstatements(); + $this->check_petscan(); + $this->check_mixnmatch(); + $this->check_reinheitsgebot(); + $this->check_listeriabot(); + $this->wrap_up(); + } + + function url_exists($url) { + return curl_init($url) !== false; + } + + function message2html_mail() { + $message = "" ; + foreach ( $this->alerts AS $tool => $tool_alerts ) { + if ( count($tool_alerts)==0 ) continue; + $message .= "

    {$tool}

    \n
      \n" ; + foreach ( $tool_alerts AS $ta ) { + $message .= "
    • {$ta['msg']}
      • \n" ; + } + $message .= "
    \n" ; + } + return $message; + } + + function message2html_page() { + $message = "

    Tool status

    " ; + foreach ( $this->alerts AS $tool => $tool_alerts ) { + $message .= "" ; + + if ( count($tool_alerts)==0 ) { + $message .= "" ; + } + $message .= "
    {$tool}Nominal"; + } else { + $message .= "" ; + foreach ( $tool_alerts AS $ta ) $message .= "
    {$ta['msg']}
    " ; + } + $message .= "
    "; + $ts = date("Y-m-d H:i:s"); + $message .= "

    Last update: {$ts}

    "; + $message .= "

    Note: If you can see an error here, Magnus has received an automated email about it, no need to contact him!

    "; + $message .= ""; + return $message; + } + + function send_message($message) { + $headers = [] ; + $headers[] = "From: Tool monitor "; + + # To send HTML mail, the Content-type header must be set + $headers[] = 'MIME-Version: 1.0'; + $headers[] = 'Content-type: text/html; charset=iso-8859-1'; + + if ( $this->testing) print "{$message}\n"; + else { + if ( mail("magnusmanske@googlemail.com","Tool monitor alert",$message,implode("\r\n", $headers),"") ) { + file_put_contents($this->last_message_file, $message); + } + } + } + + function wrap_up() { + if ( $this->testing ) print_r($this->alerts); + $message = $this->message2html_page(); + file_put_contents($this->web_page_file, $message); + + $message = $this->message2html_mail(); + if ( $message != '' ) { + $message = "\n\n{$message}\n\n"; + + if ( $this->testing ) $last_message='hni7gsib7BGUCU'; # Unique + else $last_message = file_get_contents($this->last_message_file); + if ($message!=$last_message) $this->send_message($message); + } else { + if ( !$this->testing ) file_put_contents($this->last_message_file, $message); + } + } + + function load_url($url) { + $ch = curl_init(); + $cnt = 3; + while ( $cnt>0 ) { + try { + #$ret = file_get_contents($url); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 60); + curl_setopt($ch, CURLOPT_USERAGENT, 'Magnus Tool Status/1.0'); + $ret = curl_exec($ch); + if($ret !== FALSE) return $ret; + } catch (Exception $e) { + print_r($e); + } + $cnt-- ; + sleep(5); + } + curl_close($ch); + } + + function prep_tool($name) { + $this->alerts[$name] = [] ; + return $name; + } + + + function check_quickstatements() { + $tool = $this->prep_tool("QuickStatements"); + $minutes_ago = 10 ; + $url = "https://quickstatements.toolforge.org/api.php?action=get_batches_info"; + if ( $this->url_exists($url) ) { + try { + $j = json_decode($this->load_url($url)); + $latest_edit = '0' ; + foreach ( $j->data AS $batch_id => $batch ) { + $batch = $batch->batch ; + if ( $batch->ts_created == $batch->ts_last_change ) continue ; # Just created, doesn't count as activity + if ( $latest_edit*1<$batch->ts_last_change*1 ) $latest_edit = $batch->ts_last_change; + } + $latest_edit_ts = strtotime($latest_edit); + $latest_ts_nice = date("Y-m-d H:i:s",$latest_edit_ts); + $now = time()-60*$minutes_ago; + if ( $now*1>$latest_edit_ts*1 ) $this->alerts[$tool][] = ["msg"=>"Last batch edit was more than {$minutes_ago} minutes ago ({$latest_ts_nice})"]; + } catch (Exception $e) { + $this->alerts[$tool][] = ["msg"=>"API returns non-JSON"]; + } + } else $this->alerts[$tool][] = ["msg"=>"Website does not answer"]; + } + + function check_petscan() { + $tool = $this->prep_tool("PetScan"); + if ( !$this->url_exists("https://petscan.wmflabs.org/") ) $this->alerts[$tool][] = ["msg"=>"Website does not answer"]; + } + + function check_mixnmatch() { + $tool = $this->prep_tool("Mix'n'match"); + $minutes_ago = 60; + $url = "https://mix-n-match.toolforge.org/api.php?query=get_jobs&max=50"; + try { + $j = json_decode($this->load_url($url)); + $latest_ts = '00000000000000' ; + foreach ( $j->data AS $job ) { + if ( $job->last_ts*1>$latest_ts*1) $latest_ts = $job->last_ts; + } + $latest_edit = strtotime($latest_ts); + $latest_ts_nice = date("Y-m-d H:i:s",$latest_edit); + $ts = time()-60*$minutes_ago; + if ( $ts*1>$latest_edit*1 ) $this->alerts[$tool][] = ["msg"=>"Last job change was more than {$minutes_ago} minutes ago ({$latest_ts_nice})"]; + } catch (Exception $e) { + $this->alerts[$tool][] = ["msg"=>"Can't get job info from API"]; + } + } + + function check_reinheitsgebot() { + $tool = $this->prep_tool("Reinheitsgebot"); + $url = "https://www.wikidata.org/w/api.php?action=query&list=users&ususers=Reinheitsgebot&usprop=blockinfo&format=json" ; + try { + $j = json_decode($this->load_url($url)); + $user = $j->query->users[0]; + if ( isset($user->blockid) ) $this->alerts[$tool][] = ["msg"=>"Blocked for '{$user->blockreason}' since {$user->blockedtimestamp}"]; + } catch (Exception $e) { + $this->alerts[$tool][] = ["msg"=>"Can't get user info from API"]; + } + } + + function check_listeriabot() { + $tool = $this->prep_tool("ListeriaBot"); + $minutes_ago = 60 ; + $url = "https://en.wikipedia.org/w/api.php?action=query&list=usercontribs&ucuser=ListeriaBot&uclimit=1&format=json" ; + try { + $j = json_decode($this->load_url($url)); + $last_edit = $j->query->usercontribs[0]; + $latest_edit = strtotime($last_edit->timestamp); + $ts = time()-60*$minutes_ago; + if ( $ts*1>$latest_edit*1 ) $this->alerts[$tool][] = ["msg"=>"Last enwiki edit was more than {$minutes_ago} minutes ago"]; + } catch (Exception $e) { + $this->alerts[$tool][] = ["msg"=>"Can't get user contributions from enwiki API"]; + } + } +} + +$testing = ($argv[1]??'')=='test'; +$tm = new ToolMonitor($testing) ; +$tm->run(); + + + +?> \ No newline at end of file diff --git a/tjs b/tjs index 14ad181..c7fd356 100644 --- a/tjs +++ b/tjs @@ -1,4 +1,4 @@ toolforge-jobs run --image tf-php74 --schedule '9 9 * * *' --mem 100Mi --command 'find /data/project/magnustools/tmp/ -mtime +1 -exec rm -rf {} \;' cleanup-tmp toolforge-jobs run --image tf-php74 --schedule '37 3 * * *' --mem 2Gi --command '/data/project/magnustools/scripts/buggregator/update.php update' buggregator-update toolforge-jobs run --image tf-php74 --schedule '37 4 * * *' --mem 2Gi --command '/data/project/magnustools/scripts/buggregatorbuggregator2toolinfo.php' buggregator-toolinfo - +toolforge-jobs run --image tf-php74 --schedule '15,45 * * * *' --mem 200Mi --command '/data/project/magnustools/scripts/monitor/monitor.php' tool-monitor From 11e19400271a47c4d7e927bcb7aa95f1bfaf38a9 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Wed, 3 May 2023 09:56:53 +0000 Subject: [PATCH 34/45] misc --- public_html/php/ToolforgeCommon.php | 1 + public_html/php/common.php | 5 ++++- public_html/resources/vue/commons-thumbnail.html | 13 +++++++++---- scripts/monitor/monitor.php | 16 ++++++++++------ tjs | 2 +- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/public_html/php/ToolforgeCommon.php b/public_html/php/ToolforgeCommon.php index 04b5fd5..31a820a 100644 --- a/public_html/php/ToolforgeCommon.php +++ b/public_html/php/ToolforgeCommon.php @@ -89,6 +89,7 @@ function getWebserverForWiki ( $wiki ) /*:string*/ { if ( $wiki == 'wikidatawiki' ) return "www.wikidata.org" ; if ( $wiki == 'specieswiki' ) return "species.wikimedia.org" ; if ( $wiki == 'metawiki' ) return "meta.wikimedia.org" ; + if ( preg_match ( '/^(.+)wikisourcewiki$/' , $wiki , $m ) ) return $m[1].".wikisource.org" ; $wiki = preg_replace ( '/_/' , '-' , $wiki ) ; if ( preg_match ( '/^(.+)wiki$/' , $wiki , $m ) ) return $m[1].".wikipedia.org" ; if ( preg_match ( '/^(.+)(wik.+)$/' , $wiki , $m ) ) return $m[1].".".$m[2].".org" ; diff --git a/public_html/php/common.php b/public_html/php/common.php index 21120dc..181800e 100644 --- a/public_html/php/common.php +++ b/public_html/php/common.php @@ -48,7 +48,10 @@ function getSPARQLitems ( $cmd , $varname = 'q' ) { } # Misc -function get_server_for_lp ( $lang , $project ) { global $wrapper_tfc ; return $wrapper_tfc->getWebserverForWiki ( $wrapper_tfc->getWikiForLanguageProject ( $lang , $project ) ) ; } +function get_server_for_lp ( $lang , $project ) { global $wrapper_tfc ; + $wiki = $wrapper_tfc->getWikiForLanguageProject ( $lang , $project ) ; + return $wrapper_tfc->getWebserverForWiki ( $wiki ) ; +} function fix_language_code ( $s ) { return strtolower ( preg_replace ( '/[^a-z-]/' , '' , $s ) ) ; } function check_project_name ( $s ) { return strtolower ( preg_replace ( '/[^a-z]/' , '' , $s ) ) ; } function pre ( $d ) { print "
    " ; print_r ( $d ) ; 	print "
    " ; } diff --git a/public_html/resources/vue/commons-thumbnail.html b/public_html/resources/vue/commons-thumbnail.html index e2fa98f..22a68be 100644 --- a/public_html/resources/vue/commons-thumbnail.html +++ b/public_html/resources/vue/commons-thumbnail.html @@ -13,9 +13,14 @@ Your browser doesn't support the HTML audio tag. Be sad. - - - + + + + + + + + Don't know how to show a {{img.imageinfo[0].mediatype}} @@ -27,7 +32,7 @@ 'use strict'; Vue.component ( 'commons-thumbnail' , { - props : [ 'filename' , 'width' , 'height' , 'api' ] , + props : [ 'filename' , 'width' , 'height' , 'api' , 'nolink' ] , data : function () { return { loading:true , loaded:false , img:{} , api_url:'https://commons.wikimedia.org/w/api.php' } } , created : function () { this.checkAndLoadFilename() ; diff --git a/scripts/monitor/monitor.php b/scripts/monitor/monitor.php index c6a69c5..3d76a8f 100755 --- a/scripts/monitor/monitor.php +++ b/scripts/monitor/monitor.php @@ -45,7 +45,7 @@ function message2html_mail() { } function message2html_page() { - $message = "

    Tool status

    " ; + $message = "

    Tool status

    " ; foreach ( $this->alerts AS $tool => $tool_alerts ) { $message .= "" ; @@ -75,7 +75,10 @@ function send_message($message) { if ( $this->testing) print "{$message}\n"; else { - if ( mail("magnusmanske@googlemail.com","Tool monitor alert",$message,implode("\r\n", $headers),"") ) { + $ts = "Updated: ".date("Y-m-d H:i:s"); + $message2 = str_replace("", "

    {$ts}

    ", $message); + $message2 = str_replace("", "

    {$ts}

    ", $message2); + if ( mail("magnusmanske@googlemail.com","Tool monitor alert",$message2,implode("\r\n", $headers),"") ) { file_put_contents($this->last_message_file, $message); } } @@ -92,7 +95,9 @@ function wrap_up() { if ( $this->testing ) $last_message='hni7gsib7BGUCU'; # Unique else $last_message = file_get_contents($this->last_message_file); - if ($message!=$last_message) $this->send_message($message); + if ($message!=$last_message) { + $this->send_message($message); + } } else { if ( !$this->testing ) file_put_contents($this->last_message_file, $message); } @@ -103,7 +108,6 @@ function load_url($url) { $cnt = 3; while ( $cnt>0 ) { try { - #$ret = file_get_contents($url); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 60); @@ -127,7 +131,7 @@ function prep_tool($name) { function check_quickstatements() { $tool = $this->prep_tool("QuickStatements"); - $minutes_ago = 10 ; + $minutes_ago = 60 ; $url = "https://quickstatements.toolforge.org/api.php?action=get_batches_info"; if ( $this->url_exists($url) ) { try { @@ -186,7 +190,7 @@ function check_reinheitsgebot() { function check_listeriabot() { $tool = $this->prep_tool("ListeriaBot"); - $minutes_ago = 60 ; + $minutes_ago = 60*8 ; # 8h $url = "https://en.wikipedia.org/w/api.php?action=query&list=usercontribs&ucuser=ListeriaBot&uclimit=1&format=json" ; try { $j = json_decode($this->load_url($url)); diff --git a/tjs b/tjs index c7fd356..2afeb85 100644 --- a/tjs +++ b/tjs @@ -1,4 +1,4 @@ toolforge-jobs run --image tf-php74 --schedule '9 9 * * *' --mem 100Mi --command 'find /data/project/magnustools/tmp/ -mtime +1 -exec rm -rf {} \;' cleanup-tmp toolforge-jobs run --image tf-php74 --schedule '37 3 * * *' --mem 2Gi --command '/data/project/magnustools/scripts/buggregator/update.php update' buggregator-update toolforge-jobs run --image tf-php74 --schedule '37 4 * * *' --mem 2Gi --command '/data/project/magnustools/scripts/buggregatorbuggregator2toolinfo.php' buggregator-toolinfo -toolforge-jobs run --image tf-php74 --schedule '15,45 * * * *' --mem 200Mi --command '/data/project/magnustools/scripts/monitor/monitor.php' tool-monitor +toolforge-jobs run --image bullseye --schedule '15,45 * * * *' --mem 200Mi --command '/data/project/magnustools/scripts/monitor/monitor.php' tool-monitor From b3cdb13b9877306f17f08d79a505d932be8807d2 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Wed, 25 Oct 2023 10:21:47 +0000 Subject: [PATCH 35/45] misc --- public_html/php/Widar.php | 21 +++++++++++++++------ public_html/php/oauth.php | 1 + public_html/resources/vue/wd-link.html | 25 +++++++++++++------------ 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/public_html/php/Widar.php b/public_html/php/Widar.php index 8dfb5f0..07c1cb9 100644 --- a/public_html/php/Widar.php +++ b/public_html/php/Widar.php @@ -231,7 +231,7 @@ public function process_request ( $parameter_name = 'action' ) { # Returns true if output was written, false otherwise public function render_reponse ( $botmode = true , $parameter_name = 'action' ) { # Bot output - $out = [ 'error' => 'OK' , 'data' => [] ] ; + $out = [ 'error' => 'OK' , 'status' => 'OK' , 'data' => [] ] ; $ret = false ; $callback = $this->tfc->getRequest('callback2','') ; # For botmode @@ -517,7 +517,7 @@ public function sdc_tag ( $json , $summary = '' ) { } public function get_rights () { - //$this->ensureAuth() ; + // $this->ensureAuth() ; $this->result = $this->oa->getConsumerRights() ; return $this->result ; } @@ -529,13 +529,22 @@ public function logout () { public function get_username () { $rights = $this->get_rights() ; - if ( !isset($rights) ) throw new Exception ( "Not logged in" ) ; - if ( !isset($rights->query) ) throw new Exception ( "Not logged in" ) ; - if ( !isset($rights->query->userinfo) ) throw new Exception ( "Not logged in" ) ; - if ( !isset($rights->query->userinfo->name) ) throw new Exception ( "Not logged in" ) ; + if ( !isset($rights) ) throw new Exception ( "Not logged in [1]" ) ; + if ( !isset($rights->query) ) throw new Exception ( "Not logged in [2]: ".$rights->error->info ) ; + if ( !isset($rights->query->userinfo) ) throw new Exception ( "Not logged in [3]" ) ; + if ( !isset($rights->query->userinfo->name) ) throw new Exception ( "Not logged in [4]" ) ; return $rights->query->userinfo->name ; } + public function get_user_id () { + $rights = $this->get_rights() ; + if ( !isset($rights) ) throw new Exception ( "Not logged in [1]" ) ; + if ( !isset($rights->query) ) throw new Exception ( "Not logged in [2]: ".$rights->error->info ) ; + if ( !isset($rights->query->userinfo) ) throw new Exception ( "Not logged in [3]" ) ; + if ( !isset($rights->query->userinfo->id) ) throw new Exception ( "Not logged in [4]" ) ; + return $rights->query->userinfo->id ; + } + protected function set_q_or_claim ( &$claim , $id , $qualifier_claim ) { if ( $qualifier_claim == '' ) $claim['q'] = $id ; else $claim['claim'] = $qualifier_claim ; diff --git a/public_html/php/oauth.php b/public_html/php/oauth.php index ffb7157..a7a3b6a 100644 --- a/public_html/php/oauth.php +++ b/public_html/php/oauth.php @@ -952,6 +952,7 @@ function createItemFromPage ( $site , $page ) { 'token' => $token, 'bot' => 1 ] ; + if ( !isset($summary) ) $summary=''; $this->setToolTag($params,$summary); if ( isset ( $_REQUEST['test'] ) ) { diff --git a/public_html/resources/vue/wd-link.html b/public_html/resources/vue/wd-link.html index 527e7f5..201270d 100644 --- a/public_html/resources/vue/wd-link.html +++ b/public_html/resources/vue/wd-link.html @@ -49,16 +49,14 @@ Vue.component ( 'wd-desc' , { props : [ 'item' , 'autodesc_fallback' , 'autodesc_first' , 'wd' , 'language' ] , template : '#wd-desc-template' , - data : function () { return { wikidata:{} , loaded:false , description:'' , use_autodesc:false , show_autodesc:false } } , + data : function () { return { loaded:false , description:'' , use_autodesc:false , show_autodesc:false } } , created : function () { let me = this ; if ( me.autodesc_first*1>0 ) me.show_autodesc = true ; if ( me.autodesc_first || me.autodesc_fallback ) me.use_autodesc = true ; - if ( typeof me.wd == 'undefined' ) me.wikidata = wd_link_wd ; - else me.wikidata = me.wd ; if ( me.getItemID() == '' ) return ; // No item - me.wikidata.getItemBatch ( [me.getItemID()] , function () { - let i = me.wikidata.getItem ( me.getItemID() ) ; + me.wikidata().getItemBatch ( [me.getItemID()] , function () { + let i = me.wikidata().getItem ( me.getItemID() ) ; if ( typeof i == 'undefined' ) return ; me.description = i.getDesc ( me.language ) ; // Show autodesc if fallback and manual description is empty @@ -67,6 +65,9 @@ } ) ; } , methods : { + wikidata : function () { + return this.wd ?? wd_link_wd; + } , getItemID : function () { let me = this ; if ( typeof me.item == 'undefined' ) return '' ; @@ -80,12 +81,9 @@ Vue.component ( 'wd-link' , { template : '#wd-link-template' , props : [ 'item' , 'wd' , 'label' , 'as_text' , 'smallq' , 'language' , 'string_prop' ] , - data : function () { return { use_label:'' , wikidata:{} , url:'' , string_prop_value:'' } } , + data : function () { return { use_label:'' , url:'' , string_prop_value:'' } } , created : function () { let me = this ; - if ( typeof me.wd == 'undefined' ) me.wikidata = wd_link_wd ; - else me.wikidata = me.wd ; - if ( typeof me.item == 'undefined' || me.item == '' ) return ; me.use_label = me.getItemID() ; me.url = wd_link_base + me.getItemTitle() ; @@ -93,9 +91,9 @@ me.use_label = me.label ; return ; } -//if ( typeof me.wikidata.getItem ( me.getItemID() ) == 'undefined' ) console.log ( "LOADING " + me.getItemID() ) ; - me.wikidata.getItemBatch ( [me.getItemID()] , function () { - let i = me.wikidata.getItem ( me.getItemID() ) ; + + me.wikidata().getItemBatch ( [me.getItemID()] , function () { + let i = me.wikidata().getItem ( me.getItemID() ) ; if ( typeof i == 'undefined' ) return ; if ( typeof me.language == 'undefined' ) me.use_label = i.getLabel () ; else me.use_label = i.getLabel ( me.language ) ; @@ -105,6 +103,9 @@ } ) ; } , methods : { + wikidata : function () { + return this.wd ?? wd_link_wd; + } , getItemID : function () { let me = this ; if ( typeof me.item == 'undefined' ) return '' ; From e422af98bdcb52a01fc48a0ff3297b0ad7e3822c Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Wed, 25 Oct 2023 13:09:45 +0100 Subject: [PATCH 36/45] local testing setup --- public_html/php/ToolforgeCommon.php | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/public_html/php/ToolforgeCommon.php b/public_html/php/ToolforgeCommon.php index 31a820a..462d321 100644 --- a/public_html/php/ToolforgeCommon.php +++ b/public_html/php/ToolforgeCommon.php @@ -25,6 +25,8 @@ final class ToolforgeCommon { public /*string*/ $tool_user_name ; # force different DB user name public $use_db_cache = true ; + private $is_local = false; + private $browser_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:57.0) Gecko/20100101 Firefox/57.0" ; private $db_servers = [ 'fast' => '.web.db.svc.eqiad.wmflabs' , @@ -32,7 +34,7 @@ final class ToolforgeCommon { ] ; private $cookiejar ; # For doPostRequest - private/*string*/ $mysql_user , $mysql_password ; + private/*string*/ $mysql_user , $mysql_password; private $db_cache = [] ; public function __construct ( /*string*/ $toolname = '' ) { @@ -157,8 +159,8 @@ function getDBname ( $language , $project ) /*:string*/ { private function getDBpassword () /*:string*/ { if ( isset ( $this->tool_user_name ) and $this->tool_user_name != '' ) $user = $this->tool_user_name ; else $user = str_replace ( 'tools.' , '' , get_current_user() ) ; - $passwordfile = '/data/project/' . $user . '/replica.my.cnf' ; - if ( $user == 'magnus' ) $passwordfile = '/home/' . $user . '/replica.my.cnf' ; // Command-line usage + $passwordfile = getenv("HOME").'/replica.my.cnf' ; + // if ( $user == 'magnus' ) $passwordfile = '/home/' . $user . '/replica.my.cnf' ; // Command-line usage $config = parse_ini_file( $passwordfile ); if ( isset( $config['user'] ) ) { $this->mysql_user = $config['user']; @@ -166,18 +168,22 @@ private function getDBpassword () /*:string*/ { if ( isset( $config['password'] ) ) { $this->mysql_password = $config['password']; } + $this->is_local = $config['local']=='true'; } # Requires a replica.trove.my.cnf file with user, password, and server public function openDBtrove ( $dbname = '' ) { if ( isset ( $this->tool_user_name ) and $this->tool_user_name != '' ) $user = $this->tool_user_name ; else $user = str_replace ( 'tools.' , '' , get_current_user() ) ; - $passwordfile = "/data/project/${user}/replica.trove.my.cnf" ; + $passwordfile = "/data/project/{$user}/replica.trove.my.cnf" ; $config = parse_ini_file( $passwordfile ); $user = $config['user']; $password = $config['password']; $server = $config['host']; - $db = @new mysqli($server, $user, $password , $dbname); + $this->is_local = $config['local']=='true'; + if ( $this->is_local ) $port = 3308 ; + else $port = null; + $db = @new mysqli($server, $user, $password , $dbname, $port); assert ( $db->connect_errno == 0 , 'Unable to connect to database [' . $db->connect_error . ']' ) ; return $db ; } @@ -209,10 +215,14 @@ public function openDB ( $language , $project , $slow_queries = false , $persist $this->getDBpassword() ; $dbname = $this->getDBname ( $language , $project ) ; + if ( $this->is_local ) $port = 3309 ; + else $port = null; + # Try optimal server $server = substr( $dbname, 0, -2 ) . ( $slow_queries ? $this->db_servers['slow'] : $this->db_servers['fast'] ) ; if ( $persistent ) $server = "p:$server" ; - $db = @new mysqli($server, $this->mysql_user, $this->mysql_password , $dbname); + if ( $this->is_local ) $server = "127.0.0.1"; + $db = @new mysqli($server, $this->mysql_user, $this->mysql_password , $dbname, $port); if ( $db->connect_errno > 0 and preg_match ( '/max_user_connections/' , $db->connect_error ) ) { // Bloody Toolforge DB connection limit $seconds = rand ( 10 , 60*10 ) ; // Random delay From a55b325c3854c86257caa3019e01ced07b54e2c2 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Wed, 25 Oct 2023 13:33:51 +0100 Subject: [PATCH 37/45] local testing setup --- public_html/php/ToolforgeCommon.php | 99 +++++++++++++++-------------- 1 file changed, 53 insertions(+), 46 deletions(-) diff --git a/public_html/php/ToolforgeCommon.php b/public_html/php/ToolforgeCommon.php index 462d321..22246e0 100644 --- a/public_html/php/ToolforgeCommon.php +++ b/public_html/php/ToolforgeCommon.php @@ -14,6 +14,13 @@ //if ( !isset($noheaderwhatsoever) ) header("Connection: close"); /* +Local testing: +ssh magnus@tools-login.wmflabs.org -L 3309:wikidatawiki.web.db.svc.eqiad.wmflabs:3306 -N & +ssh magnus@tools-login.wmflabs.org -L 3308:tools-db:3306 -N & +*/ + +/* +USAGE: require_once ( '/data/project/magnustools/public_html/php/ToolforgeCommon.php' ) ; $tfc = new ToolforgeCommon('mytoolname') ; */ @@ -25,17 +32,18 @@ final class ToolforgeCommon { public /*string*/ $tool_user_name ; # force different DB user name public $use_db_cache = true ; - private $is_local = false; + private $isLocal = false; - private $browser_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:57.0) Gecko/20100101 Firefox/57.0" ; - private $db_servers = [ + private $browserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:57.0) Gecko/20100101 Firefox/57.0" ; + private $dbServers = [ 'fast' => '.web.db.svc.eqiad.wmflabs' , 'slow' => '.analytics.db.svc.eqiad.wmflabs' ] ; private $cookiejar ; # For doPostRequest - private/*string*/ $mysql_user , $mysql_password; - private $db_cache = [] ; + private/*string*/ $mysqlUser; + private/*string*/ $mysqlPassword; + private $dbCache = [] ; public function __construct ( /*string*/ $toolname = '' ) { if ( $toolname != '' ) $this->toolname = $toolname ; @@ -55,37 +63,37 @@ private function determineToolname () /*:string*/ { // CONVENIENCE - function getRequest ( $key , $default = "" ) /*:string*/ { + public function getRequest ( $key , $default = "" ) /*:string*/ { if ( isset ( $this->prefilled_requests[$key] ) ) return $this->prefilled_requests[$key] ; if ( isset ( $_REQUEST[$key] ) ) return str_replace ( "\'" , "'" , $_REQUEST[$key] ) ; return $default ; } - function urlEncode ( $t ) /*:string*/ { + public function urlEncode ( $t ) /*:string*/ { $t = str_replace ( " " , "_" , $t ) ; $t = urlencode ( $t ) ; return $t ; } - function escapeAttribute ( $s ) /*:string*/ { - $ret = preg_replace ( "/\"/" , '"' , $s ) ; - $ret = preg_replace ( "/'/" , ''' , $ret ) ; + public function escapeAttribute ( $s ) /*:string*/ { + $ret = str_replace ( "\"" , '"' , $s ) ; + $ret = str_replace ( "'" , ''' , $ret ) ; return $ret ; } - function flush () /*:void*/ { + public function flush () /*:void*/ { @ob_flush(); flush(); } - function getCurrentTimestamp () /*:string*/ { + public function getCurrentTimestamp () /*:string*/ { return date ( 'YmdHis' ) ; } // Toolforge/Wikimedia name conversions - function getWebserverForWiki ( $wiki ) /*:string*/ { + public function getWebserverForWiki ( $wiki ) /*:string*/ { $wiki = preg_replace ( '/_p$/' , '' , $wiki ) ; // Paranoia if ( $wiki == 'commonswiki' ) return "commons.wikimedia.org" ; if ( $wiki == 'wikidatawiki' ) return "www.wikidata.org" ; @@ -98,7 +106,7 @@ function getWebserverForWiki ( $wiki ) /*:string*/ { return '' ; } - function trimWikitextMarkup ( $wikitext ) /*:string*/ { + public function trimWikitextMarkup ( $wikitext ) /*:string*/ { $wikitext = preg_replace ( '/\[\[(.+?)[\#\|].*?\]\]/' , '$1' , $wikitext ) ; $wikitext = preg_replace ( '/\[\[(.+?)\]\]/' , '$1' , $wikitext ) ; $wikitext = preg_replace ( '/\{\{(.+?)\}\}/' , '' , $wikitext ) ; @@ -106,7 +114,7 @@ function trimWikitextMarkup ( $wikitext ) /*:string*/ { return $wikitext ; } - function getWikiPageText ( $wiki , $page ) /*:string*/ { + public function getWikiPageText ( $wiki , $page ) /*:string*/ { $server = $this->getWebserverForWiki ( $wiki ) ; $url = "https://{$server}/w/api.php?action=parse&prop=wikitext&format=json&page=" . $this->urlEncode ( $page ) ; $json = json_decode ( @file_get_contents ( $url ) ) ; @@ -118,11 +126,10 @@ function getWikiPageText ( $wiki , $page ) /*:string*/ { $ret = $json->parse->wikitext->$star ; $ret = @iconv('UTF-8', 'UTF-8//IGNORE', $ret) ; -// print "$ret\n" ; exit(0) ; return $ret ; } - function getWikiForLanguageProject ( $language , $project ) /*:string*/ { + public function getWikiForLanguageProject ( $language , $project ) /*:string*/ { $language = trim ( strtolower ( $language ) ) ; if ( $language == 'commons' ) return 'commonswiki' ; if ( $language == 'wikidata' ) return 'wikidatawiki' ; @@ -134,7 +141,7 @@ function getWikiForLanguageProject ( $language , $project ) /*:string*/ { // DATABASE - function getDBname ( $language , $project ) /*:string*/ { + public function getDBname ( $language , $project ) /*:string*/ { $ret = $language ; if ( $language == 'commons' ) $ret = 'commonswiki_p' ; elseif ( $language == 'wikidata' || $project == 'wikidata' ) $ret = 'wikidatawiki_p' ; @@ -163,12 +170,12 @@ private function getDBpassword () /*:string*/ { // if ( $user == 'magnus' ) $passwordfile = '/home/' . $user . '/replica.my.cnf' ; // Command-line usage $config = parse_ini_file( $passwordfile ); if ( isset( $config['user'] ) ) { - $this->mysql_user = $config['user']; + $this->mysqlUser = $config['user']; } if ( isset( $config['password'] ) ) { - $this->mysql_password = $config['password']; + $this->mysqlPassword = $config['password']; } - $this->is_local = $config['local']=='true'; + $this->isLocal = $config['local']=='true'; } # Requires a replica.trove.my.cnf file with user, password, and server @@ -180,9 +187,7 @@ public function openDBtrove ( $dbname = '' ) { $user = $config['user']; $password = $config['password']; $server = $config['host']; - $this->is_local = $config['local']=='true'; - if ( $this->is_local ) $port = 3308 ; - else $port = null; + $this->isLocal = $config['local']=='true'; $db = @new mysqli($server, $user, $password , $dbname, $port); assert ( $db->connect_errno == 0 , 'Unable to connect to database [' . $db->connect_error . ']' ) ; return $db ; @@ -192,11 +197,15 @@ public function openDBtool ( $dbname = '' , $server = '' , $force_user = '' , $p $this->getDBpassword() ; if ( $dbname == '' ) $dbname = '_main' ; else $dbname = "__$dbname" ; - if ( $force_user == '' ) $dbname = $this->mysql_user.$dbname; + if ( $force_user == '' ) $dbname = $this->mysqlUser.$dbname; else $dbname = $force_user.$dbname; - if ( $server == '' ) $server = "tools.labsdb" ; //"tools-db" ; + if ( $server == '' ) $server = "tools.labsdb" ; if ( $persistent ) $server = "p:$server" ; - $db = @new mysqli($server, $this->mysql_user, $this->mysql_password , $dbname); + if ( $this->isLocal ) { + $port = 3308 ; + $server = "127.0.0.1"; + } else $port = null; + $db = @new mysqli($server, $this->mysqlUser, $this->mysqlPassword , $dbname, $port); assert ( $db->connect_errno == 0 , 'Unable to connect to database [' . $db->connect_error . ']' ) ; return $db ; } @@ -210,21 +219,22 @@ public function openDBwiki ( $wiki , $slow_queries = false , $persistent = false public function openDB ( $language , $project , $slow_queries = false , $persistent = false ) { $db_key = "$language.$project" ; - if ( !$persistent and isset ( $this->db_cache[$db_key] ) ) return $this->db_cache[$db_key] ; + if ( !$persistent and isset ( $this->dbCache[$db_key] ) ) return $this->dbCache[$db_key] ; $this->getDBpassword() ; $dbname = $this->getDBname ( $language , $project ) ; - if ( $this->is_local ) $port = 3309 ; + if ( $this->isLocal ) $port = 3309 ; else $port = null; # Try optimal server - $server = substr( $dbname, 0, -2 ) . ( $slow_queries ? $this->db_servers['slow'] : $this->db_servers['fast'] ) ; + $server = substr( $dbname, 0, -2 ) . ( $slow_queries ? $this->dbServers['slow'] : $this->dbServers['fast'] ) ; if ( $persistent ) $server = "p:$server" ; - if ( $this->is_local ) $server = "127.0.0.1"; - $db = @new mysqli($server, $this->mysql_user, $this->mysql_password , $dbname, $port); + if ( $this->isLocal ) $server = "127.0.0.1"; + $db = @new mysqli($server, $this->mysqlUser, $this->mysqlPassword , $dbname, $port); - if ( $db->connect_errno > 0 and preg_match ( '/max_user_connections/' , $db->connect_error ) ) { // Bloody Toolforge DB connection limit + // Coding around low Toolforge DB connection limit + if ( $db->connect_errno > 0 and preg_match ( '/max_user_connections/' , $db->connect_error ) ) { $seconds = rand ( 10 , 60*10 ) ; // Random delay sleep ( $seconds ) ; return $this->openDB ( $language , $project , $slow_queries , $persistent ) ; @@ -232,13 +242,13 @@ public function openDB ( $language , $project , $slow_queries = false , $persist # Try the other server if($db->connect_errno > 0 ) { - $server = substr( $dbname, 0, -2 ) . ( $slow_queries ? $this->db_servers['fast'] : $this->db_servers['slow'] ) ; + $server = substr( $dbname, 0, -2 ) . ( $slow_queries ? $this->dbServers['fast'] : $this->dbServers['slow'] ) ; if ( $persistent ) $server = "p:$server" ; - $db = @new mysqli($server, $this->mysql_user, $this->mysql_password , $dbname); + $db = @new mysqli($server, $this->mysqlUser, $this->mysqlPassword , $dbname); } assert ( $db->connect_errno == 0 , 'Unable to connect to database [' . $db->connect_error . ']' ) ; - if ( !$persistent and $this->use_db_cache ) $this->db_cache[$db_key] = $db ; + if ( !$persistent and $this->use_db_cache ) $this->dbCache[$db_key] = $db ; return $db ; } @@ -247,7 +257,6 @@ public function getSQL ( &$db , &$sql , $max_tries = 5 , $message = '' ) { $pinging = 10 ; while ( !@$db->ping() ) { if ( $pinging < 0 ) break ; - // print "RECONNECTING..." ; sleep ( 5/$max_tries ) ; @$db->connect() ; $pinging-- ; @@ -269,7 +278,7 @@ public function findSubcats ( &$db , $root , &$subcats , $depth = -1 ) { $subcats[$r] = $db->real_escape_string ( $r ) ; $c[] = $db->real_escape_string ( $r ) ; } - if ( count ( $c ) == 0 ) return ; + if ( empty($c) ) return ; if ( $depth == 0 ) return ; $sql = "SELECT DISTINCT page_title FROM page,categorylinks WHERE page_id=cl_from AND cl_to IN ('" . implode ( "','" , $c ) . "') AND cl_type='subcat'" ; $result = $this->getSQL ( $db , $sql , 2 ) ; @@ -277,7 +286,7 @@ public function findSubcats ( &$db , $root , &$subcats , $depth = -1 ) { if ( isset ( $subcats[$row['page_title']] ) ) continue ; $check[] = $row['page_title'] ; } - if ( count ( $check ) == 0 ) return ; + if ( empty($check) ) return ; $this->findSubcats ( $db , $check , $subcats , $depth - 1 ) ; } @@ -348,7 +357,7 @@ public function getCommonFooter () /*:string*/ { // CURL // Takes a URL and an array with POST parameters - function doPostRequest ( $url , $params = [] , $optional_headers = null ) { + public function doPostRequest ( $url , $params = [] , $optional_headers = null ) { $ch = curl_init(); curl_setopt($ch, CURLOPT_COOKIEJAR, $this->cookiejar); curl_setopt($ch, CURLOPT_COOKIEFILE, $this->cookiejar); @@ -356,7 +365,7 @@ function doPostRequest ( $url , $params = [] , $optional_headers = null ) { curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_POST, true); curl_setopt($ch, CURLOPT_POSTFIELDS, $params); - curl_setopt($ch, CURLOPT_USERAGENT, $this->browser_agent); + curl_setopt($ch, CURLOPT_USERAGENT, $this->browserAgent); if ($optional_headers !== null) curl_setopt($ch, CURLOPT_HTTPHEADER, $optional_headers); $output = curl_exec($ch); $info = curl_getinfo($ch); @@ -385,7 +394,7 @@ public function getMultipleURLsInParallel ( $urls , $batch_size = 50 ) { $ch[$key] = curl_init($value); // curl_setopt($ch[$key], CURLOPT_NOBODY, true); // curl_setopt($ch[$key], CURLOPT_HEADER, true); - curl_setopt($ch[$key], CURLOPT_USERAGENT, $this->browser_agent); + curl_setopt($ch[$key], CURLOPT_USERAGENT, $this->browserAgent); curl_setopt($ch[$key], CURLOPT_RETURNTRANSFER, true); curl_setopt($ch[$key], CURLOPT_SSL_VERIFYPEER, false); curl_setopt($ch[$key], CURLOPT_SSL_VERIFYHOST, false); @@ -547,8 +556,7 @@ public function runCommandsQS ( $commands , $qs = '' ) { $tmp['data']['commands'] = $qs->compressCommands ( $tmp_uncompressed['data']['commands'] ) ; $qs->runCommandArray ( $tmp['data']['commands'] ) ; if ( !isset($qs->last_item) ) return ; - $last_item = 'Q' . preg_replace ( '/\D/' , '' , "{$qs->last_item}" ) ; - return $last_item ; + return 'Q' . preg_replace ( '/\D/' , '' , "{$qs->last_item}" ) ; } private function guessConfigFile () /*:string*/ { @@ -569,8 +577,7 @@ public function logToolUse ( $toolname = '' , $method = '' ) { } $url = 'https://tools.wmflabs.org/magnustools/logger.php?tool=' . urlencode($toolname) ; if ( trim($method) != '' ) $url .= '&method=' . urlencode(trim($method)) ; - $j = json_decode ( file_get_contents ( $url ) ) ; - return $j ; + return json_decode ( file_get_contents ( $url ) ) ; } public function showProcInfo() { From 56cc15bf8432775193a66c63a8e99796b35eec42 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Wed, 25 Oct 2023 15:07:19 +0100 Subject: [PATCH 38/45] misc --- public_html/php/ToolforgeCommon.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/public_html/php/ToolforgeCommon.php b/public_html/php/ToolforgeCommon.php index 22246e0..57b3d05 100644 --- a/public_html/php/ToolforgeCommon.php +++ b/public_html/php/ToolforgeCommon.php @@ -166,8 +166,9 @@ public function getDBname ( $language , $project ) /*:string*/ { private function getDBpassword () /*:string*/ { if ( isset ( $this->tool_user_name ) and $this->tool_user_name != '' ) $user = $this->tool_user_name ; else $user = str_replace ( 'tools.' , '' , get_current_user() ) ; - $passwordfile = getenv("HOME").'/replica.my.cnf' ; - // if ( $user == 'magnus' ) $passwordfile = '/home/' . $user . '/replica.my.cnf' ; // Command-line usage + // $passwordfile = getenv("HOME").'/replica.my.cnf' ; + $passwordfile = "/data/project/{$user}/replica.trove.my.cnf" ; + if ( $user == 'magnus' ) $passwordfile = '/home/' . $user . '/replica.my.cnf' ; // Command-line usage $config = parse_ini_file( $passwordfile ); if ( isset( $config['user'] ) ) { $this->mysqlUser = $config['user']; @@ -188,7 +189,7 @@ public function openDBtrove ( $dbname = '' ) { $password = $config['password']; $server = $config['host']; $this->isLocal = $config['local']=='true'; - $db = @new mysqli($server, $user, $password , $dbname, $port); + $db = @new mysqli($server, $user, $password , $dbname); assert ( $db->connect_errno == 0 , 'Unable to connect to database [' . $db->connect_error . ']' ) ; return $db ; } From 18c917c0c7fa23a09d5155a73e1d6d070493bcc9 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Wed, 25 Oct 2023 15:08:05 +0100 Subject: [PATCH 39/45] misc --- public_html/php/ToolforgeCommon.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public_html/php/ToolforgeCommon.php b/public_html/php/ToolforgeCommon.php index 57b3d05..197b1fe 100644 --- a/public_html/php/ToolforgeCommon.php +++ b/public_html/php/ToolforgeCommon.php @@ -167,7 +167,7 @@ private function getDBpassword () /*:string*/ { if ( isset ( $this->tool_user_name ) and $this->tool_user_name != '' ) $user = $this->tool_user_name ; else $user = str_replace ( 'tools.' , '' , get_current_user() ) ; // $passwordfile = getenv("HOME").'/replica.my.cnf' ; - $passwordfile = "/data/project/{$user}/replica.trove.my.cnf" ; + $passwordfile = "/data/project/{$user}/replica.my.cnf" ; if ( $user == 'magnus' ) $passwordfile = '/home/' . $user . '/replica.my.cnf' ; // Command-line usage $config = parse_ini_file( $passwordfile ); if ( isset( $config['user'] ) ) { From 6b45d45dd3a3614b78bb621db51fe6fa02313258 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Wed, 25 Oct 2023 15:08:51 +0100 Subject: [PATCH 40/45] misc --- public_html/php/ToolforgeCommon.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public_html/php/ToolforgeCommon.php b/public_html/php/ToolforgeCommon.php index 197b1fe..9fc8555 100644 --- a/public_html/php/ToolforgeCommon.php +++ b/public_html/php/ToolforgeCommon.php @@ -176,7 +176,7 @@ private function getDBpassword () /*:string*/ { if ( isset( $config['password'] ) ) { $this->mysqlPassword = $config['password']; } - $this->isLocal = $config['local']=='true'; + $this->isLocal = isset($config['local'])&&($config['local']=='true'); } # Requires a replica.trove.my.cnf file with user, password, and server From e4abdb6b3bedfdde0325df513b272e1972b1af36 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Wed, 1 Nov 2023 13:44:10 +0000 Subject: [PATCH 41/45] misc --- scripts/buggregator/Buggregator.php | 30 +++++++++++++++++++++++++++++ scripts/buggregator/update.php | 1 + scripts/monitor/monitor.php | 14 +++++++++++++- tjs | 3 ++- 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/scripts/buggregator/Buggregator.php b/scripts/buggregator/Buggregator.php index 5bcf199..2f06949 100644 --- a/scripts/buggregator/Buggregator.php +++ b/scripts/buggregator/Buggregator.php @@ -843,6 +843,36 @@ public function get_or_create_text_id ( $text ) { return $this->last_insert_id() ; } + public function update_http_status() { + // Reset status to unknown + $sql = "UPDATE `tool` SET `http_status`=null WHERE `subdomain`=''" ; + $this->getSQL ( $sql ) ; + + // Get current http status + $sql = "SELECT DISTINCT `subdomain` FROM `tool` WHERE `subdomain`!=''" ; + $result = $this->getSQL ( $sql ) ; + while($o = $result->fetch_object()) { + $url = "https://{$o->subdomain}.toolforge.org" ; + $output = trim(shell_exec("curl -sI \"{$url}\" | grep HTTP | head -1")); + if ( preg_match('|^HTTP/2\s*(\d+)$|',$output,$m) ) { + $status = $m[1]*1; + $sql = "UPDATE `tool` SET `http_status`={$status} WHERE `subdomain`='{$o->subdomain}'" ; + } else { + // print "Could not run curl for {$url}\n"; + $sql = "UPDATE `tool` SET `http_status`=null WHERE `subdomain`='{$o->subdomain}'" ; + } + $this->getSQL ( $sql ); + } + } + + public function get_broken_subdomains() { + $ret = []; + $sql = "SELECT DISTINCT `subdomain` FROM `tool` WHERE `has_webservice`=1 AND (`http_status`!=200 OR `http_status` IS NULL)"; + $result = $this->getSQL ( $sql ) ; + while($o = $result->fetch_object()) $ret[] = $o->subdomain; + return $ret; + } + } ?> \ No newline at end of file diff --git a/scripts/buggregator/update.php b/scripts/buggregator/update.php index cd4ae82..ba1333f 100755 --- a/scripts/buggregator/update.php +++ b/scripts/buggregator/update.php @@ -14,6 +14,7 @@ $issue->update_in_database ( $buggregator , ['status'] ) ; $buggregator->touch_issue($issue->id()); } +else if ( $argv[1] == 'webstatus' ) $buggregator->update_http_status(); #else if ( $argv[1] == 'test' ) $buggregator->check_wikidata_for_tool_items() ; else print "Nothing to do!\n" ; diff --git a/scripts/monitor/monitor.php b/scripts/monitor/monitor.php index 3d76a8f..f971dc0 100755 --- a/scripts/monitor/monitor.php +++ b/scripts/monitor/monitor.php @@ -4,7 +4,8 @@ error_reporting(E_ERROR|E_CORE_ERROR|E_ALL|E_COMPILE_ERROR); ini_set('display_errors', 'On'); -require_once('/data/project/magnustools/public_html/php/ToolforgeCommon.php') ; +// require_once('/data/project/magnustools/public_html/php/ToolforgeCommon.php') ; +require_once('/data/project/magnustools/scripts/buggregator/Buggregator.php') ; class ToolMonitor { public $tfc ; @@ -24,9 +25,20 @@ public function run() { $this->check_mixnmatch(); $this->check_reinheitsgebot(); $this->check_listeriabot(); + $this->check_buggregator(); $this->wrap_up(); } + function check_buggregator() { + $buggregator = new Buggregator ; + $subdomains = $buggregator->get_broken_subdomains(); + foreach ($subdomains as $subdomain) { + if ( in_array($subdomain, ['quickstatements','petscan','mix-n-match','listeriabot']) ) continue; + if ( isset($this->alerts[$subdomain]) ) continue; + $this->alerts[$subdomain][] = ["msg"=>"Buggregator reports non-200 status"]; + } + } + function url_exists($url) { return curl_init($url) !== false; } diff --git a/tjs b/tjs index 2afeb85..7315f8d 100644 --- a/tjs +++ b/tjs @@ -1,4 +1,5 @@ toolforge-jobs run --image tf-php74 --schedule '9 9 * * *' --mem 100Mi --command 'find /data/project/magnustools/tmp/ -mtime +1 -exec rm -rf {} \;' cleanup-tmp toolforge-jobs run --image tf-php74 --schedule '37 3 * * *' --mem 2Gi --command '/data/project/magnustools/scripts/buggregator/update.php update' buggregator-update -toolforge-jobs run --image tf-php74 --schedule '37 4 * * *' --mem 2Gi --command '/data/project/magnustools/scripts/buggregatorbuggregator2toolinfo.php' buggregator-toolinfo +toolforge-jobs run --image tf-php74 --schedule '37 4 * * *' --mem 2Gi --command '/data/project/magnustools/scripts/buggregator/buggregator2toolinfo.php' buggregator-toolinfo toolforge-jobs run --image bullseye --schedule '15,45 * * * *' --mem 200Mi --command '/data/project/magnustools/scripts/monitor/monitor.php' tool-monitor +toolforge-jobs run --image php8.2 --schedule '5,35 * * * *' --mem 200Mi --command '/data/project/magnustools/scripts/buggregator/update.php webstatus' buggregator-http-status From 5b8cea412000072a2c8753023c11472a4ac11ef5 Mon Sep 17 00:00:00 2001 From: Magnus Manske Date: Wed, 24 Jan 2024 11:16:18 +0000 Subject: [PATCH 42/45] misc --- public_html/php/oauth.php | 3 + public_html/resources/js/wikidata.js | 12 +++ .../resources/vue/commons-thumbnail.html | 20 ++--- .../resources/vue/mastodon-button.html | 78 +++++++++++++++++++ public_html/resources/vue/wd-link.html | 2 +- 5 files changed, 104 insertions(+), 11 deletions(-) create mode 100644 public_html/resources/vue/mastodon-button.html diff --git a/public_html/php/oauth.php b/public_html/php/oauth.php index a7a3b6a..5dd6fed 100644 --- a/public_html/php/oauth.php +++ b/public_html/php/oauth.php @@ -1,5 +1,8 @@ - ... + ... - + - + - + - + @@ -32,8 +32,8 @@ 'use strict'; Vue.component ( 'commons-thumbnail' , { - props : [ 'filename' , 'width' , 'height' , 'api' , 'nolink' ] , - data : function () { return { loading:true , loaded:false , img:{} , api_url:'https://commons.wikimedia.org/w/api.php' } } , + props : [ 'filename' , 'width' , 'height' , 'api' , 'nolink' , 'videothumbnail' , 'loading' ] , + data : function () { return { is_loading:true , loaded:false , img:{} , api_url:'https://commons.wikimedia.org/w/api.php' } } , created : function () { this.checkAndLoadFilename() ; } , @@ -41,7 +41,7 @@ checkAndLoadFilename : function () { let me = this ; if ( typeof me.filename == 'undefined' ) { - me.loading = false ; + me.is_loading = false ; me.loaded = false ; return ; } @@ -60,8 +60,8 @@ me.img = v ; me.loaded = true ; } ) ; - // console.log ( JSON.parse(JSON.stringify(me.img)) ) ; - me.loading = false ; + // console.log ( JSON.parse(JSON.stringify(me.img)) ) ; + me.is_loading = false ; } ) ; } , getElementStyle : function () { diff --git a/public_html/resources/vue/mastodon-button.html b/public_html/resources/vue/mastodon-button.html new file mode 100644 index 0000000..2abd878 --- /dev/null +++ b/public_html/resources/vue/mastodon-button.html @@ -0,0 +1,78 @@ + + + + + + diff --git a/public_html/resources/vue/wd-link.html b/public_html/resources/vue/wd-link.html index 201270d..7458f1e 100644 --- a/public_html/resources/vue/wd-link.html +++ b/public_html/resources/vue/wd-link.html @@ -5,7 +5,7 @@ LAST - {{use_label}} + {{use_label}} {{use_label}} From 8970d644b131c4f16ea185a699876a9276a7b3f5 Mon Sep 17 00:00:00 2001 From: dena Date: Thu, 16 Jan 2025 14:03:03 +0100 Subject: [PATCH 43/45] test --- public_html/php/WbstackMagnusOauth.php | 1 - 1 file changed, 1 deletion(-) diff --git a/public_html/php/WbstackMagnusOauth.php b/public_html/php/WbstackMagnusOauth.php index cad971a..8364610 100644 --- a/public_html/php/WbstackMagnusOauth.php +++ b/public_html/php/WbstackMagnusOauth.php @@ -172,7 +172,6 @@ public static function getSite( // XXX: this same logic is in quickstatements.php and platform api WikiController backend $domain = $_SERVER['SERVER_NAME']; if ( self::isLocalHost() ){ - // localhost development, with a full domain prefixing .localhost // eg. wiki.addshore.com.localhost $wbRoot = $domain; From 34c987514f9332468884ddee3ed7881ee6eab3fa Mon Sep 17 00:00:00 2001 From: dena Date: Thu, 16 Jan 2025 14:47:13 +0100 Subject: [PATCH 44/45] DEBUG --- public_html/php/WbstackMagnusOauth.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public_html/php/WbstackMagnusOauth.php b/public_html/php/WbstackMagnusOauth.php index 8364610..debd144 100644 --- a/public_html/php/WbstackMagnusOauth.php +++ b/public_html/php/WbstackMagnusOauth.php @@ -181,7 +181,7 @@ public static function getSite( $publicMwOAuthUrl = 'http://' . $domain . '/w/index.php?title=Special:OAuth'; $mwOAuthUrl = 'http://' . self::platformIngressHostAndPort . '/w/index.php?title=Special:OAuth'; $wbPublicHostAndPort = $wbRoot; - $wbApi = 'http://' . self::platformIngressHostAndPort . '/w/api.php'; + $wbApi = 'TEST http://' . self::platformIngressHostAndPort . '/w/api.php'; $wbPageBase = $wbRoot . '/wiki/'; $toolbase = $toolRoot; $entityBase = 'http://' . $wbRoot . '/entity/'; From c7454d7d90d33a8b62f529184dcf1941373e9780 Mon Sep 17 00:00:00 2001 From: dena Date: Thu, 16 Jan 2025 14:49:22 +0100 Subject: [PATCH 45/45] add gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..22d0d82 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +vendor
    {$tool}