From 329c215beddd2c124b2d8fe86e3f273beeb46cd5 Mon Sep 17 00:00:00 2001 From: Chris Schnaufer Date: Wed, 13 Mar 2019 13:12:07 -0700 Subject: [PATCH 01/36] Moving database maintenance scripts to bety project --- script/reindex.bety.sh | 150 ++++++++++++++++++++++++++++++ script/vacuum.bety.sh | 204 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 354 insertions(+) create mode 100755 script/reindex.bety.sh create mode 100755 script/vacuum.bety.sh diff --git a/script/reindex.bety.sh b/script/reindex.bety.sh new file mode 100755 index 000000000..03c9f696d --- /dev/null +++ b/script/reindex.bety.sh @@ -0,0 +1,150 @@ +#!/bin/bash + +# exit on error +set -e + +# ---------------------------------------------------------------------- +# START CONFIGURATION SECTION +# ---------------------------------------------------------------------- + +# name of the dabase to reindexing +# this script assumes the user running it has access to the database +DATABASE=${DATABASE:-"bety"} + +# The database catalog to search for mactching table names +DBCATALOG=${DBCATALOG:-"bety"} + +# psql options +# this allows you to add the user to use as well as any other options +PG_OPT=${PG_OPT-"-U bety"} + +# Should the process be quiet +QUIET=${QUIET:-"NO"} + +# Should all the tables be indexed +TABLENAME=${TABLENAME:-"ALL"} + +# Non-database-system tables that should always be ignored on a reindex +IGNORETABLES=${IGNORETABLES:-""} + +# Skip reindexing the entire database after reindexing the tables +SKIPDATABASE=${SKIPDATABASE:-"YES"} + +# ---------------------------------------------------------------------- +# END CONFIGURATION SECTION +# ---------------------------------------------------------------------- + +# parse command line options +while getopts c:d:hi:p:qst: opt; do + case $opt in + c) + DBCATALOG=$OPTARG + ;; + d) + DATABASE=$OPTARG + ;; + h) + echo "$0 [-c datalog] [-d database] [-h] [-i table names] [-p psql options] [-q] [-s] [-t tablename]" + echo " -c catalog, database catalog name used to search for tables, default is bety" + echo " -d database, default is bety" + echo " -h this help page" + echo " -i table names, list of space-separated table names to skip over when reindexing" + echo " -p additional psql command line options, default is -U bety" + echo " -q the reindexing should be quiet" + echo " -s reindex the database after reindexing the tables (this should be done sparingly)" + echo " -t tablename, the name of the one table to reindex" + exit 0 + ;; + i) + # We add spaces to assist in exact table name maching + IGNORETABLES=" ${OPTARG} " + ;; + p) + PG_OPT=$OPTARG + ;; + q) + QUIET="YES" + ;; + s) + SKIPDATABASE="NO" + ;; + t) + TABLENAME=$OPTARG + SKIPDATABASE="YES" + ;; + esac +done + +# be quiet if not interactive +if ! tty -s ; then + exec 1>/dev/null +fi + +# find current schema version +# following returns a triple: +# - number of migrations +# - largest migration +# - hash of all migrations +MIGRATIONS=$( psql ${PG_OPT} -t -q -d "${DATABASE}" -c 'SELECT COUNT(version) FROM schema_migrations' | tr -d ' ' ) +VERSION=$( psql ${PG_OPT} -t -q -d "${DATABASE}" -c 'SELECT md5(array_agg(version)::text) FROM (SELECT version FROM schema_migrations ORDER BY version) as v;' | tr -d ' ' ) +LATEST=$( psql ${PG_OPT} -t -q -d "${DATABASE}" -c 'SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1' | tr -d ' ' ) +NOW=$( date -u +"%Y-%m-%dT%H:%M:%SZ" ) +if [ "${QUIET}" != "YES" ]; then + echo "Version: ${MIGRATIONS} ${VERSION} ${LATEST}" + echo "Starting reindexing: ${NOW}" +fi + +# Inform the caller of what we're doing +if [ "${QUIET}" != "YES" ]; then + RUNINFO="" + if [ "${TABLENAME}" == "ALL" ]; then + RUNINFO="Reindexing all tables in catalog ${DBCATALOG}" + else + RUNINFO="Reindexing ${TABLENAME}" + fi + if [ "${SKIPDATABASE}" == "YES" ]; then + RUNINFO="${RUNINFO}, skipping entire database" + else + RUNINFO="${RUNINFO}, reindexing entire database" + fi + echo "${RUNINFO}" +fi + +# If we are reindexing all the tables, get the list of tables from the database +if [ "${TABLENAME}" == "ALL" ]; then + TABLENAME=$(psql ${PG_OPT} -t -q -d "${DATABASE}" -c "with curuser as (select user) select t.table_name from information_schema.tables t join pg_catalog.pg_class c on (t.table_name = c.relname) join pg_catalog.pg_user u on (c.relowner = u.usesysid) join curuser cu on (u.usename = cu.current_user) where t.table_catalog='${DBCATALOG}' and t.table_schema='public' and t.table_type like '%TABLE%' order by t.table_name asc" | tr -d ' ') + if [ "${QUIET}" != "YES" ]; then + printf "Reindexing all tables\n" + fi +else + if [ "${QUIET}" != "YES" ]; then + printf "Reindexing %-40s\n" "${TABLENAME}" + fi +fi + +# Reindex the tables +for T in ${TABLENAME}; do + if echo "${IGNORETABLES}" | grep -qi " ${T} "; then + if [ "${QUIET}" != "YES" ]; then + printf "Ignoring %-40s\n" "${T}" + fi + else + if [ "${QUIET}" != "YES" ]; then + printf "Reindex %-40s\n" "${T}" + fi + psql ${PG_OPT} -t -q -d "${DATABASE}" -c "REINDEX TABLE ${T}" + fi +done + +# Reindexing the overall database, should be fast(er) with all the tables already reindexed +if [ "${SKIPDATABASE}" == "NO" ]; then + if [ "${QUIET}" != "YES" ]; then + printf "Reindex entire database\n" + fi + psql ${PG_OPT} -t -q -d "${DATABASE}" -c "REINDEX DATABASE ${DATABASE}" +fi + +if [ "${QUIET}" != "YES" ]; then + NOW=$( date -u +"%Y-%m-%dT%H:%M:%SZ" ) + echo "Completed reindexing: ${NOW}" +fi diff --git a/script/vacuum.bety.sh b/script/vacuum.bety.sh new file mode 100755 index 000000000..7fff9af9e --- /dev/null +++ b/script/vacuum.bety.sh @@ -0,0 +1,204 @@ +#!/bin/bash + +# exit on error +set -e + +# ---------------------------------------------------------------------- +# START CONFIGURATION SECTION +# ---------------------------------------------------------------------- + +# name of the dabase to vacuum +# this script assumes the user running it has access to the database +DATABASE=${DATABASE:-"bety"} + +# The database catalog to search for mactching table names +DBCATALOG=${DBCATALOG:-"bety"} + +# Perform a full vacuum which returns resources to the system +FULLVACUUM=${FULLVACUUM:-"NO"} + +# psql options +# this allows you to add the user to use as well as any other options +PG_OPT=${PG_OPT-"-U bety"} + +# Should the process be quiet +QUIET=${QUIET:-"NO"} + +# Should all the tables be vacuumed +TABLENAME=${TABLENAME:-"ALL"} + +# Whether the tables should be analyzed +ANALYZETABLES=${ANALYZETABLES:-"YES"} + +# Only analyze the tables, don't perform a regular vacuum +ANALYZEONLY=${ANALYZEONLY:-"NO"} + +# Non-database-system tables that should always be ignored on a vacuum +IGNORETABLES=${IGNORETABLES:-""} + +# Skip vacuuming the entire database after vacuuming the tables +SKIPDATABASE=${SKIPDATABASE:-"NO"} + +# ---------------------------------------------------------------------- +# END CONFIGURATION SECTION +# ---------------------------------------------------------------------- + +# parse command line options +while getopts c:d:fhi:np:qst:z opt; do + case $opt in + c) + DBCATALOG=$OPTARG + ;; + d) + DATABASE=$OPTARG + ;; + f) + if [ "${ANALYZEONLY}" == "NO" ]; then + FULLVACUUM="YES" + fi + ;; + h) + echo "$0 [-c datalog] [-d database] [-f] [-h] [-i table names] [-n] [-p psql options] [-q] [-s] [-t tablename] [-z]" + echo " -c catalog, database catalog name used to search for tables, default is bety" + echo " -d database, default is bety" + echo " -f perform a full vacuum to return resources to the system. Specify rarely, if ever" + echo " -h this help page" + echo " -i table names, list of space-separated table names to skip over when vacuuming" + echo " -n only vacuum the tables and do not analyze, default is to first vacuum and then analyze" + echo " -p additional psql command line options, default is -U bety" + echo " -q the vacuum should be quiet" + echo " -s skip vacuuming the database after vacuuming the tables" + echo " -t tablename, the name of the one table to vacuum" + echo " -z only perform analyze, do not perform a regular vacuum, overrides -n and -f, sets -s" + exit 0 + ;; + i) + # We add spaces to assist in exact table name maching + IGNORETABLES=" ${OPTARG} " + ;; + n) + if [ "${ANALYZEONLY}" == "NO" ]; then + ANALYZETABLES="NO" + fi + ;; + p) + PG_OPT=$OPTARG + ;; + q) + QUIET="YES" + ;; + s) + SKIPDATABASE="YES" + ;; + t) + TABLENAME=$OPTARG + SKIPDATABASE="YES" + ;; + z) + ANALYZEONLY="YES" + ANALYZETABLES="YES" + SKIPDATABASE="YES" + FULLVACUUM="NO" + ;; + esac +done + +# be quiet if not interactive +if ! tty -s ; then + exec 1>/dev/null +fi + +# find current schema version +# following returns a triple: +# - number of migrations +# - largest migration +# - hash of all migrations +MIGRATIONS=$( psql ${PG_OPT} -t -q -d "${DATABASE}" -c 'SELECT COUNT(version) FROM schema_migrations' | tr -d ' ' ) +VERSION=$( psql ${PG_OPT} -t -q -d "${DATABASE}" -c 'SELECT md5(array_agg(version)::text) FROM (SELECT version FROM schema_migrations ORDER BY version) as v;' | tr -d ' ' ) +LATEST=$( psql ${PG_OPT} -t -q -d "${DATABASE}" -c 'SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1' | tr -d ' ' ) +NOW=$( date -u +"%Y-%m-%dT%H:%M:%SZ" ) +if [ "${QUIET}" != "YES" ]; then + echo "Version: ${MIGRATIONS} ${VERSION} ${LATEST}" + echo "Starting vacuum: ${NOW}" +fi + +# Inform the caller of what we're doing +if [ "${QUIET}" != "YES" ]; then + RUNINFO="" + if [ "${TABLENAME}" == "ALL" ]; then + RUNINFO="Vacuuming all tables in catalog ${DBCATALOG}" + else + RUNINFO="Vacuuming ${TABLENAME}" + fi + if [ "${ANALYZETABLES}" == "YES" ]; then + RUNINFO="${RUNINFO}, with analysis" + else + RUNINFO="${RUNINFO}, skipping analysis" + fi + if [ "${SKIPDATABASE}" == "YES" ]; then + RUNINFO="${RUNINFO}, skipping entire database" + fi + if [ "${FULLVACUUM}" == "YES" ]; then + RUNINFO="${RUNINFO}, with the FULL option set (this may take a long time)" + fi + echo "${RUNINFO}" +fi + +# If we are vacuuming all the tables, get the list of tables from the database +if [ "${TABLENAME}" == "ALL" ]; then + TABLENAME=$(psql ${PG_OPT} -t -q -d "${DATABASE}" -c "select table_name from information_schema.tables where table_catalog='${DBCATALOG}' and table_schema='public' and table_type like '%TABLE%' order by table_name asc" | tr -d ' ') + if [ "${QUIET}" != "YES" ]; then + printf "Vacuuming all tables\n" + fi +else + if [ "${QUIET}" != "YES" ]; then + printf "Vacuuming %-40s\n" "${TABLENAME}" + fi +fi + +# Vacuum the tables +FULLOPTION="" +if [ "${FULLVACUUM}" == "YES" ]; then + FULLOPTION="FULL" +fi +for T in ${TABLENAME}; do + if echo "${IGNORETABLES}" | grep -qi " ${T} "; then + if [ "${QUIET}" != "YES" ]; then + printf "Ignoring %-40s\n" "${T}" + fi + else + if [ "${ANALYZEONLY}" == "NO" ]; then + if [ "${QUIET}" != "YES" ]; then + printf "Vacuum %s%-40s\n" "${FULLOPTION} " "${T}" + fi + psql ${PG_OPT} -t -q -d "${DATABASE}" -c "VACUUM ${FULLOPTION} ${T}" + fi + if [ "${ANALYZETABLES}" == "YES" ]; then + if [ "${QUIET}" != "YES" ]; then + printf "Vacuum analyze %-40s\n" "${T}" + fi + psql ${PG_OPT} -t -q -d "${DATABASE}" -c "VACUUM ANALYZE ${T}" + fi + fi +done + +# Vacuum the overall database, should be fast(er) with all the tables already vacuumed +if [ "${SKIPDATABASE}" == "NO" ]; then + if [ "${ANALYZEONLY}" == "NO" ]; then + if [ "${QUIET}" != "YES" ]; then + printf "Vacuum %sdatabase\n" "${FULLOPTION} " + fi + psql ${PG_OPT} -t -q -d "${DATABASE}" -c "VACUUM ${FULLOPTION}" + fi + if [ "${ANALYZETABLES}" == "YES" ]; then + if [ "${QUIET}" != "YES" ]; then + printf "Vacuum analyze database\n" + fi + psql ${PG_OPT} -t -q -d "${DATABASE}" -c "VACUUM ANALYZE" + fi +fi + +if [ "${QUIET}" != "YES" ]; then + NOW=$( date -u +"%Y-%m-%dT%H:%M:%SZ" ) + echo "Completed vacuum: ${NOW}" +fi From 454e543060ddf8eee6010f618464d914337bff94 Mon Sep 17 00:00:00 2001 From: Chris Schnaufer Date: Wed, 13 Mar 2019 13:29:40 -0700 Subject: [PATCH 02/36] Added reindex and vacuum options to script, minor typo fixes --- docker/entrypoint.sh | 25 +++++++++++++++++++++++++ script/reindex.bety.sh | 4 ++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 3c598558b..7ba293587 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -23,6 +23,14 @@ case $1 in echo "Migrate database." rake db:migrate SKIP_SCHEMASPY=YES ;; + "reindex" ) + echo "Reindexing database tables" + ./script/reindex.bety.sh -d "bety" -p "-h postgres -p 5432 -U postgres" + ;; + "reindex-all" ) + echo "Reindexing entire database" + ./script/reindex.bety.sh -d "bety" -p "-h postgres -p 5432 -U postgres" -s + ;; "server" ) echo "Start running BETY (rails server)" exec rails s @@ -35,6 +43,18 @@ case $1 in echo "Start running BETY (unicorn)" exec bundle exec unicorn -c config/unicorn.rb ;; + "vacuum" ) + echo "Vacuuming database tables" + ./script/vacuum.bety.sh -d "bety" -p "-h postgres -p 5432 -U postgres" -s + ;; + "vacuum-all" ) + echo "Vacuuming entire database (not VACUUM FULL)" + ./script/vacuum.bety.sh -d "bety" -p "-h postgres -p 5432 -U postgres" + ;; + "vacuum-full" ) + echo "Full vacuuming of entire database: VACUUM FULL" + ./script/vacuum.bety.sh -d "bety" -p "-h postgres -p 5432 -U postgres" -f + ;; "autoserver" ) echo "Migrate database." rake db:migrate SKIP_SCHEMASPY=YES @@ -50,8 +70,13 @@ case $1 in echo "sync : synchronize database with remote servers ${REMOTE_SERVERS}" echo "dump : dumps local database" echo "migrate : migrates the database to a new version of bety" + echo "reindex : maintentance: reindex the tables in the database" + echo "reindex-all: maintentance: reindex all of the database, do this sparingly" echo "server : runs the server (using rails server)" echo "unicorn : runs the server (using unicorn)" + echo "vacuum : maintenance: vaccum the tables of the database" + echo "vacuum-all : maintenance: vaccum the entire database (not VACUUM FULL)" + echo "vacuum-full: maintenance: full vaccum of the database. Specify rarely, if ever" echo "autoserver : runs the server (using unicorn) after running a migrate" echo "help : this text" echo "" diff --git a/script/reindex.bety.sh b/script/reindex.bety.sh index 03c9f696d..d6c1fc876 100755 --- a/script/reindex.bety.sh +++ b/script/reindex.bety.sh @@ -44,7 +44,7 @@ while getopts c:d:hi:p:qst: opt; do DATABASE=$OPTARG ;; h) - echo "$0 [-c datalog] [-d database] [-h] [-i table names] [-p psql options] [-q] [-s] [-t tablename]" + echo "$0 [-c catalog] [-d database] [-h] [-i table names] [-p psql options] [-q] [-s] [-t tablename]" echo " -c catalog, database catalog name used to search for tables, default is bety" echo " -d database, default is bety" echo " -h this help page" @@ -56,7 +56,7 @@ while getopts c:d:hi:p:qst: opt; do exit 0 ;; i) - # We add spaces to assist in exact table name maching + # We add spaces to assist in exact table name matching IGNORETABLES=" ${OPTARG} " ;; p) From 8dbdff9a56ba12d59684e11038ec9e6a9d70fa3a Mon Sep 17 00:00:00 2001 From: Chris Schnaufer Date: Thu, 14 Mar 2019 13:27:54 -0700 Subject: [PATCH 03/36] Updated book keeping files --- .zenodo.json | 4 ++++ CHANGELOG.md | 1 + 2 files changed, 5 insertions(+) diff --git a/.zenodo.json b/.zenodo.json index 80a703a6f..94972177b 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -49,6 +49,10 @@ }, { "name": "Chris Black" + }, + { + "affiliation": "University of Arizona", + "name": "Chris Schnaufer" } ], "language": "eng", diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c1b1326d..3eda714de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ For more information about this file see also [Keep a Changelog](http://keepacha - #505 : Added a new attributes table. This table allows to store through polymorphism any additional information with any row in any table. The value stored is a json document. - #597 : Moved `dump.bety.sh` and `load.bety.sh` from PEcAn to BETY. +- #643 : Moved `reindex.bety.sh` and `vacuum.bety.sh` scripts from PEcAn to BETY and provided access through entrypoint. ## [5.1.0] - 2019-01-14 From bb3c76acee4f39aea1e7fc289a31caaabbbec2db Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Thu, 14 Mar 2019 20:55:24 -0500 Subject: [PATCH 04/36] Created Unreleased section and move comment there. --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 021d6343a..232d4a517 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ section for the next release. For more information about this file see also [Keep a Changelog](http://keepachangelog.com/) . +## [Unreleased] + +- #643 : Moved `reindex.bety.sh` and `vacuum.bety.sh` scripts from PEcAn to BETY and provided access through entrypoint. + ## [5.2.0] - 2019-03-11 ### Added @@ -12,7 +16,6 @@ For more information about this file see also [Keep a Changelog](http://keepacha - Ability to set the initialize URL in docker - #505 : Added a new attributes table. This table allows to store through polymorphism any additional information with any row in any table. The value stored is a json document. - #597 : Moved `dump.bety.sh` and `load.bety.sh` from PEcAn to BETY. -- #643 : Moved `reindex.bety.sh` and `vacuum.bety.sh` scripts from PEcAn to BETY and provided access through entrypoint. ## [5.1.0] - 2019-01-14 From b29610e765d787706b284cdd495a20fc0cc5e698 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Thu, 23 May 2019 11:47:28 +0000 Subject: [PATCH 05/36] Bump rubyzip from 1.2.1 to 1.2.2 Bumps [rubyzip](https://github.com/rubyzip/rubyzip) from 1.2.1 to 1.2.2. - [Release notes](https://github.com/rubyzip/rubyzip/releases) - [Changelog](https://github.com/rubyzip/rubyzip/blob/master/Changelog.md) - [Commits](https://github.com/rubyzip/rubyzip/compare/v1.2.1...v1.2.2) --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 2d9206e10..a5a87b6be 100644 --- a/Gemfile +++ b/Gemfile @@ -22,7 +22,7 @@ gem 'rails3-restful-authentication', '~> 3.0.1' gem 'dynamic_form' gem 'rabl' gem 'yajl-ruby', '~> 1.3.1' -gem 'rubyzip', '~> 1.2.1' +gem 'rubyzip', '~> 1.2.2' gem 'activerecord-session_store' # no longer part of Rails proper gem 'protected_attributes_continued' # Use this until and unless we start using Strong Parameters. gem 'sass-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 00000d1ae..6bf38186a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -202,7 +202,7 @@ GEM mustache (~> 1.0, >= 0.99.4) rspec (~> 3.0) ruby-graphviz (1.0.8) - rubyzip (1.2.1) + rubyzip (1.2.2) safe_attributes (1.0.10) activerecord (>= 3.0.0) sass (3.5.6) @@ -286,7 +286,7 @@ DEPENDENCIES rspec-rails (~> 3.0) rspec_api_documentation ruby-graphviz (= 1.0.8) - rubyzip (~> 1.2.1) + rubyzip (~> 1.2.2) safe_attributes sass-rails seer (= 0.10.0) From 1f25b45f4538ec1e22bdc6017cd7aa6eee8d09be Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Thu, 23 May 2019 11:47:36 +0000 Subject: [PATCH 06/36] Bump loofah from 2.2.2 to 2.2.3 Bumps [loofah](https://github.com/flavorjones/loofah) from 2.2.2 to 2.2.3. - [Release notes](https://github.com/flavorjones/loofah/releases) - [Changelog](https://github.com/flavorjones/loofah/blob/master/CHANGELOG.md) - [Commits](https://github.com/flavorjones/loofah/compare/v2.2.2...v2.2.3) --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 00000d1ae..8ceb79803 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -100,7 +100,7 @@ GEM json-schema (2.8.0) addressable (>= 2.4) kgio (2.11.2) - loofah (2.2.2) + loofah (2.2.3) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.0) From e2263f6c1eaec22f75c3376ad859991f2daa47ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Thu, 23 May 2019 11:48:17 +0000 Subject: [PATCH 07/36] Bump nokogiri from 1.8.2 to 1.8.5 Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.8.2 to 1.8.5. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/master/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.8.2...v1.8.5) --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 00000d1ae..2b9198c63 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -114,7 +114,7 @@ GEM mustache (1.0.5) narray (0.6.0.4) nio4r (2.3.0) - nokogiri (1.8.2) + nokogiri (1.8.5) mini_portile2 (~> 2.3.0) passenger (5.3.4) rack From c7a355bc39948b93991f079f2dcb8e18334086ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Thu, 23 May 2019 11:48:32 +0000 Subject: [PATCH 08/36] Bump rack from 2.0.5 to 2.0.7 Bumps [rack](https://github.com/rack/rack) from 2.0.5 to 2.0.7. - [Release notes](https://github.com/rack/rack/releases) - [Changelog](https://github.com/rack/rack/blob/master/CHANGELOG.md) - [Commits](https://github.com/rack/rack/compare/2.0.5...2.0.7) --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 00000d1ae..aba054d4f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -133,7 +133,7 @@ GEM public_suffix (3.0.2) rabl (0.13.1) activesupport (>= 2.3.14) - rack (2.0.5) + rack (2.0.7) rack-test (1.0.0) rack (>= 1.0, < 3) railroad (0.5.0) From 2874fc5105ff17dcb27779112c356120dd8584f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Thu, 23 May 2019 13:51:35 +0000 Subject: [PATCH 09/36] Bump ffi from 1.9.23 to 1.11.1 Bumps [ffi](https://github.com/ffi/ffi) from 1.9.23 to 1.11.1. - [Release notes](https://github.com/ffi/ffi/releases) - [Changelog](https://github.com/ffi/ffi/blob/master/CHANGELOG.md) - [Commits](https://github.com/ffi/ffi/compare/1.9.23...1.11.1) --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index aba054d4f..d74fd1c3d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -85,7 +85,7 @@ GEM dynamic_form (1.1.4) erubi (1.7.1) execjs (2.7.0) - ffi (1.9.23) + ffi (1.11.1) globalid (0.4.1) activesupport (>= 4.2.0) i18n (0.9.5) From fe38e2b1c0572ec590f3f46f018c69f4b773315f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2019 15:47:50 +0000 Subject: [PATCH 10/36] Bump yard from 0.9.12 to 0.9.20 Bumps [yard](https://github.com/lsegal/yard) from 0.9.12 to 0.9.20. - [Release notes](https://github.com/lsegal/yard/releases) - [Changelog](https://github.com/lsegal/yard/blob/master/CHANGELOG.md) - [Commits](https://github.com/lsegal/yard/compare/v0.9.12...v0.9.20) Signed-off-by: dependabot[bot] --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 482987915..3c12699da 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -245,7 +245,7 @@ GEM xpath (3.0.0) nokogiri (~> 1.8) yajl-ruby (1.3.1) - yard (0.9.12) + yard (0.9.20) PLATFORMS ruby From 3a318978f9bc81c36bb1252d51487f71a6a4858e Mon Sep 17 00:00:00 2001 From: Scott Rohde Date: Thu, 18 Jul 2019 15:48:48 -0500 Subject: [PATCH 11/36] Modified site soil choices (c.f. issue #652). --- app/models/site.rb | 13 ++++++++++++- app/views/sites/edit.html.erb | 2 +- app/views/sites/new.html.erb | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/app/models/site.rb b/app/models/site.rb index 5803ce960..e2bf92e22 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -231,7 +231,7 @@ def city_state def sitename_state_country output = "" - + #city = city.chomp if !city.nil? if !sitename.blank? output += "#{sitename}" @@ -262,6 +262,17 @@ def autocomplete_label "#{sitename.squish} (#{city.squish}, #{!(state.nil? || state.empty?) ? " #{state.squish}," : ""} #{country.squish})" end + ### Class methods + + def self.soil_classes + + @soil_classes ||= ['clay', 'clay loam', 'loam', 'loamy sand', 'sand', + 'sandy clay', 'sandy clay loam', 'sandy loam', + 'silt', 'silt loam', 'silty clay', 'silty clay loam', + 'peat', 'bedrock', 'other'] + end + + private ## Validation methods diff --git a/app/views/sites/edit.html.erb b/app/views/sites/edit.html.erb index 8def26abe..9bc90cabe 100644 --- a/app/views/sites/edit.html.erb +++ b/app/views/sites/edit.html.erb @@ -311,7 +311,7 @@ <%= f.label :lon %> <%= geometry_aware_text_field(f, :lon, { class: "input-full" }) %> <%= f.label :soil %> - <%= f.select :soil, %w(sandloamy sandsandy loamsilt loamloamsandy clay loamsilty clay loamclay loamsandy claysilty clayclaypeat), { :include_blank => true }, :class => "input-full" %> + <%= f.select :soil, Site.soil_classes, { :include_blank => true }, :class => "input-full" %>
<%= f.label :clay_pct, "% Clay" %> diff --git a/app/views/sites/new.html.erb b/app/views/sites/new.html.erb index 7aa7b9607..da66b2a18 100644 --- a/app/views/sites/new.html.erb +++ b/app/views/sites/new.html.erb @@ -309,7 +309,7 @@ <%= f.text_field :lon, :class => "input-full" %>

<%= f.label :soil %> - <%= f.select :soil, %w(sandloamy sandsandy loamsilt loamloamsandy clay loamsilty clay loamclay loamsandy claysilty clayclaypeat), { :include_blank => true }, :class => "input-full" %> + <%= f.select :soil, Site.soil_classes, { :include_blank => true }, :class => "input-full" %>
<%= f.label :clay_pct, "% Clay" %> From 1d9cf043cd274b0f814b3d45a19b7f429ad7bdfd Mon Sep 17 00:00:00 2001 From: Scott Rohde Date: Thu, 18 Jul 2019 16:04:49 -0500 Subject: [PATCH 12/36] Make the @soil_classes array immutable. --- app/models/site.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/site.rb b/app/models/site.rb index e2bf92e22..e41690137 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -269,7 +269,7 @@ def self.soil_classes @soil_classes ||= ['clay', 'clay loam', 'loam', 'loamy sand', 'sand', 'sandy clay', 'sandy clay loam', 'sandy loam', 'silt', 'silt loam', 'silty clay', 'silty clay loam', - 'peat', 'bedrock', 'other'] + 'peat', 'bedrock', 'other'].freeze end From 66da7496d63b92afb58803085df8fe3e61bdf2b2 Mon Sep 17 00:00:00 2001 From: Scott Rohde Date: Tue, 23 Jul 2019 10:17:24 -0500 Subject: [PATCH 13/36] Changed links to BETYdb data-entry and technical guides to point to bookdown versions. --- app/views/layouts/application.html.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index ce1bf2c95..03dd6273b 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -200,9 +200,9 @@
  • Docs
      -
    • <%= link_to "Data Entry", "https://pecan.gitbooks.io/betydbdoc-dataentry/content/", :target =>"_blank" %>
      • +
    • <%= link_to "Data Entry", "https://pecanproject.github.io/bety-documentation/dataentry/", :target =>"_blank" %>
    • <%= link_to "Data Access", "https://pecan.gitbooks.io/betydb-data-access/content/", :target =>"_blank" %>>
      • -
    • <%= link_to "Implementation", "https://pecan.gitbooks.io/betydb-documentation/content/", :target =>"_blank" %>
      • +
    • <%= link_to "Implementation", "https://pecanproject.github.io/bety-documentation/technical", :target =>"_blank" %>
    • <%= link_to "Schema", schemas_path %>>
    • From e1976d25d43f6411fcccdae58f41dfc23be28851 Mon Sep 17 00:00:00 2001 From: Scott Rohde Date: Wed, 31 Jul 2019 14:34:12 -0500 Subject: [PATCH 14/36] Updating capybara-webkit to version 1.15.1. This may solve installation problems the cropped up when upgrading to MacOS Mojave and may also solve problems in the Travis build. --- Gemfile.lock | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 482987915..5d4ff424b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -63,7 +63,7 @@ GEM rack (>= 1.0.0) rack-test (>= 0.5.4) xpath (>= 2.0, < 4.0) - capybara-webkit (1.15.0) + capybara-webkit (1.15.1) capybara (>= 2.3, < 4.0) json childprocess (0.9.0) @@ -298,3 +298,6 @@ DEPENDENCIES will_paginate yajl-ruby (~> 1.3.1) yard + +BUNDLED WITH + 1.17.3 From 246ec12f2c884b1ab2b51db847fe68a2db369291 Mon Sep 17 00:00:00 2001 From: David LeBauer Date: Mon, 5 Aug 2019 11:56:30 -0700 Subject: [PATCH 15/36] update documentation links reflecting migration to bookdown --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 00485bd36..962a30224 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ docker run -e RAILS_RELATIVE_URL_ROOT="/bety" pecan/bety ## Documentation. -* Technical Documentation: https://pecan.gitbooks.io/betydb-documentation/content/ -* Data Entry: https://pecan.gitbooks.io/betydbdoc-dataentry/content/ -* Data Access: https://pecan.gitbooks.io/betydb-data-access/content/ +* Technical Documentation: https://pecanproject.github.io/bety-documentation/technical/ +* Data Entry: https://pecanproject.github.io/bety-documentation/dataentry/ +* Data Access: https://pecan.gitbook.io/betydb-data-access/ From 5138177af3632d7b94bae30449afa5c3f3e674ee Mon Sep 17 00:00:00 2001 From: David LeBauer Date: Mon, 5 Aug 2019 11:57:42 -0700 Subject: [PATCH 16/36] Update rails_helper.rb --- spec/rails_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 7dfd03a88..1ce44b7f2 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -11,7 +11,7 @@ require 'capybara/rspec' rescue => e puts e.message - puts "Go to https://pecan.gitbooks.io/betydb-documentation/content/automated_tests.html for instructions on setting up the testing environment." + puts "Go to https://pecanproject.github.io/bety-documentation/technical/automated-tests.html for instructions on setting up the testing environment." exit end From a0ef5ca8a88bfca38b7d6c4c7b20f4e9fef91e76 Mon Sep 17 00:00:00 2001 From: David LeBauer Date: Mon, 5 Aug 2019 12:00:13 -0700 Subject: [PATCH 17/36] Update application.html.erb --- app/views/layouts/application.html.erb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index ce1bf2c95..67d8e31fc 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -200,9 +200,9 @@
  • Docs
      -
    • <%= link_to "Data Entry", "https://pecan.gitbooks.io/betydbdoc-dataentry/content/", :target =>"_blank" %>
      • -
    • <%= link_to "Data Access", "https://pecan.gitbooks.io/betydb-data-access/content/", :target =>"_blank" %>>
      • -
    • <%= link_to "Implementation", "https://pecan.gitbooks.io/betydb-documentation/content/", :target =>"_blank" %>
      • +
    • <%= link_to "Data Entry", "https://pecanproject.github.io/bety-documentation/dataentry", :target =>"_blank" %>
      • +
    • <%= link_to "Data Access", "https://pecan.gitbook.io/betydb-data-access/", :target =>"_blank" %>>
      • +
    • <%= link_to "Implementation", "https://pecanproject.github.io/bety-documentation/technical/", :target =>"_blank" %>
    • <%= link_to "Schema", schemas_path %>>
    • @@ -292,7 +292,7 @@

    BETYdb

    • <%= link_to "Homepage", root_path %>
      • -
    • <%= link_to "Documentation", "https://pecan.gitbooks.io/betydb-documentation/content/", :target =>"_blank" %>
      • +
    • <%= link_to "Documentation", "https://pecanproject.github.io/bety-documentation/", :target =>"_blank" %>
      • <% if CONFIG[:show_crop_map_links] %>
    • <%= link_to "Maps & Data", maps_path %>
      • <% end %> From 44b163760099b246ed5b7e893c5cf3fe2f27e798 Mon Sep 17 00:00:00 2001 From: David LeBauer Date: Mon, 5 Aug 2019 12:02:45 -0700 Subject: [PATCH 18/36] Update load.bety.sh --- script/load.bety.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/load.bety.sh b/script/load.bety.sh index 42b04e987..80bdfdbf5 100755 --- a/script/load.bety.sh +++ b/script/load.bety.sh @@ -250,7 +250,7 @@ ID_RANGE=1000000000 # before anything is done, check to make sure database exists if ! psql ${PG_OPT} ${PG_USER} -lqt | cut -d \| -f 1 | grep -w "^ *${DATABASE} *$" > /dev/null ; then echo "Database ${DATABASE} does not exist, please create it:" - echo "(see https://pecan.gitbooks.io/betydb-documentation/content/installing_betydb.html)" + echo "(see https://pecanproject.github.io/bety-documentation/technical/deploying-a-production-copy-of-the-betydb-web-application.html)" echo " psql ${PG_OPT} ${PG_USER} -c \"CREATE ROLE ${OWNER} WITH LOGIN CREATEDB NOSUPERUSER NOCREATEROLE PASSWORD 'password'\"" echo " psql ${PG_OPT} ${PG_USER} -c \"CREATE DATABASE ${DATABASE} WITH OWNER ${OWNER}\"" exit 1 From 34cb7114325a123b9fce8a6869cde762383d18e6 Mon Sep 17 00:00:00 2001 From: Scott Rohde Date: Tue, 6 Aug 2019 10:27:00 -0500 Subject: [PATCH 19/36] Updated change log. --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3e55ecbf..ace01eba7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ section for the next release. For more information about this file see also [Keep a Changelog](http://keepachangelog.com/) . +## [5.2.1] - 2019-08-06 + +### Fixes + +- Restores method assignment in Bulk Uploads + ## [5.2.0] - 2019-03-11 ### Added From 56c5815fe838469636607a78bdf79826f592f6f0 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Tue, 6 Aug 2019 11:02:30 -0500 Subject: [PATCH 20/36] fix travis build, ignore javascript testing --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1b206e858..00a5fc480 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,9 @@ notifications: services: - postgresql +install: + - bundle install --jobs=3 --retry=3 --with docker --without "javascript_testing" + before_script: - psql -U postgres -c "CREATE EXTENSION postgis" - psql -U postgres -c "CREATE USER bety WITH SUPERUSER CREATEDB UNENCRYPTED PASSWORD 'bety'"; From 45e26ddf7560940417c93b2ed2c7d9e4ab64b51a Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Tue, 6 Aug 2019 11:13:43 -0500 Subject: [PATCH 21/36] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ace01eba7..7ee479718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ For more information about this file see also [Keep a Changelog](http://keepacha ### Fixes - Restores method assignment in Bulk Uploads +- Fixed travis build ## [5.2.0] - 2019-03-11 From 45dea3cfc838892417a40c6e0a8f9ce9ef32f1f0 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Tue, 6 Aug 2019 14:18:58 -0500 Subject: [PATCH 22/36] install postgis 2.4 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 00a5fc480..9297bd013 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ addons: postgresql: 9.5 apt: packages: - - postgresql-9.5-postgis-2.3 + - postgresql-9.5-postgis-2.4 # notifications should go to gitter notifications: From c3f723f0c885086ca16a50155d8c38af1137612a Mon Sep 17 00:00:00 2001 From: Scott Rohde Date: Tue, 6 Aug 2019 16:33:48 -0500 Subject: [PATCH 23/36] Don't try to configure the Capybara Webkit if it's not loaded. --- spec/rails_helper.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 7dfd03a88..ef48c12ca 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -69,8 +69,10 @@ Bundler.require('javascript_testing') Capybara.javascript_driver = :webkit - Capybara::Webkit.configure do |config| - config.block_unknown_urls + if defined?(Capybara::Webkit) + Capybara::Webkit.configure do |config| + config.block_unknown_urls + end end class Binding From 7bf9e5aab3d463be435f6934dec2d7f9b21d9f26 Mon Sep 17 00:00:00 2001 From: David LeBauer Date: Tue, 6 Aug 2019 15:13:45 -0700 Subject: [PATCH 24/36] Update README --- script/db_maintenance/README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/db_maintenance/README b/script/db_maintenance/README index b0ff1275b..ea6ebd3fa 100644 --- a/script/db_maintenance/README +++ b/script/db_maintenance/README @@ -15,7 +15,7 @@ This script was motivated by the need to repair a database where the id numbers for new rows were allocated in the wrong range—to be specific, they were allocated starting at 99000000001 rather than starting at the number corresponding to the id number of the machine where the database resides. (See -https://pecan.gitbook.io/betydb-documentation/betydb-system-administration/distributed-betydb#primary-key-allocations +https://pecanproject.github.io/bety-documentation/technical/distributed-instances-of-betydb.html for a list of BETYdb instances and their corresponding primary key allocations.) When id numbers have been mis-allocated, three things must be done to repair the From 16e0d0ef6c0c800fbd7927eabfe3cfe63a9762ef Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Wed, 7 Aug 2019 10:34:58 -0500 Subject: [PATCH 25/36] fix changelog --- CHANGELOG.md | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 232d4a517..e7b2f4272 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,20 @@ For more information about this file see also [Keep a Changelog](http://keepacha ## [Unreleased] +### Added + - #643 : Moved `reindex.bety.sh` and `vacuum.bety.sh` scripts from PEcAn to BETY and provided access through entrypoint. +### Changed + +- Documentation updates, now point to new [documentation location](https://pecanproject.github.io/bety-documentation/index.html) +- #652 : Better list of soil choices + +### Fixed + +- Updated GEM list to fix security fixes as recommended by dependbot on GitHub. + + ## [5.2.0] - 2019-03-11 ### Added @@ -19,7 +31,7 @@ For more information about this file see also [Keep a Changelog](http://keepacha ## [5.1.0] - 2019-01-14 -### Fixes +### Fixed - #611 : could not close alerts - #585 : Pressing "Show" on the Covariates Edit page attempts an update @@ -30,34 +42,34 @@ For more information about this file see also [Keep a Changelog](http://keepacha ## [5.0.5] - 2018-10-23 -### Fixes +### Fixed - #603 : Restores Rails route to make method completion work in Bulk Upload Wizard ## [5.0.4] - 2018-10-11 -### Fixes +### Fixed - #600 : Error when starting BETY as docker container due to frozen variable. ## [5.0.3] - 2018-09-28 -### Fixes +### Fixed - #593 : Replaces references to gitter with references to slack. -- #598 : Fixes issue in docker where the entrypoint referenced a wrong URL to download the initial schema from. +- #598 : Fixed issue in docker where the entrypoint referenced a wrong URL to download the initial schema from. ## [5.0.2] - 2018-08-09 -### Fixes +### Fixed - Update to sprockets gem ## [5.0.1] - 2018-08-08 -### Fixes -- Fixes fuzzy matching used in search queries +### Fixed +- Fixed fuzzy matching used in search queries ## [5.0.0] - 2018-08-07 -### Fixes +### Fixed - Major upgrade to rails 5, make sure to read documentation on how to upgrade. From 112d27fd9d98caa838b8ca45c0b9ae0dc5513cc3 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Wed, 7 Aug 2019 11:12:36 -0500 Subject: [PATCH 26/36] update ruby version --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1b206e858..7e1bab798 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,10 @@ sudo: false # versions of ruby to use # see the list at http://rubies.travis-ci.org/ rvm: - - 2.2.4 - - 2.3.0 + - 2.3.8 + - 2.4.6 + - 2.5.5 + - 2.6.3 # used for all ruby tasks to specify the database env: From 169355684bbde1d99f5d5ef1a204cd4cc9bb979a Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Wed, 7 Aug 2019 11:41:14 -0500 Subject: [PATCH 27/36] ignore all rvm excetp 2.3 for now --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.travis.yml b/.travis.yml index db5746b52..2cf50f5f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,11 +7,20 @@ sudo: false # versions of ruby to use # see the list at http://rubies.travis-ci.org/ rvm: + - 2.2.10 - 2.3.8 - 2.4.6 - 2.5.5 - 2.6.3 +matrix: + fast_finish: true + allow_failures: + - rvm: 2.2.10 + - rvm: 2.4.6 + - rvm: 2.5.5 + - rvm: 2.6.3 + # used for all ruby tasks to specify the database env: - RAILS_ENV=test From 4194982c77501b99db315091e2ee3b5a4f65ee32 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Wed, 7 Aug 2019 12:10:14 -0500 Subject: [PATCH 28/36] 2.2.10 is apparently less than 2.2.2 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2cf50f5f8..a08c80fe1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ sudo: false # versions of ruby to use # see the list at http://rubies.travis-ci.org/ rvm: - - 2.2.10 + - 2.2.9 - 2.3.8 - 2.4.6 - 2.5.5 @@ -16,7 +16,7 @@ rvm: matrix: fast_finish: true allow_failures: - - rvm: 2.2.10 + - rvm: 2.2.9 - rvm: 2.4.6 - rvm: 2.5.5 - rvm: 2.6.3 From 2b8ccafafb4eeca99a8bcb93e9cc44736ea194b7 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Wed, 7 Aug 2019 14:42:19 -0500 Subject: [PATCH 29/36] updated ruby version for rvm --- .ruby-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ruby-version b/.ruby-version index a831048a2..a9eea9a34 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -ruby-2.3.0 +ruby-2.3.8 From 6e38869952a73cb85f443829b1c7dd9b41b42311 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Thu, 8 Aug 2019 10:38:06 -0500 Subject: [PATCH 30/36] Update CHANGELOG.md --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4be28ce6b..d0ec47bfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ For more information about this file see also [Keep a Changelog](http://keepacha ## [5.2.1] - 2019-08-07 +Currently BETY does not compile correctly with any newer version of Ruby than 2.3. We are aware of this issue and tracking it #664. + ### Added - #643 : Moved `reindex.bety.sh` and `vacuum.bety.sh` scripts from PEcAn to BETY and provided access through entrypoint. @@ -15,12 +17,13 @@ For more information about this file see also [Keep a Changelog](http://keepacha - Documentation updates, now point to new [documentation location](https://pecanproject.github.io/bety-documentation/index.html) - #652 : Better list of soil choices +- Ruby 2.3.8 is now default ruby when using RVM. ### Fixed - Updated GEM list to fix security fixes as recommended by dependbot on GitHub. - Restores method assignment in Bulk Uploads -- Fixed travis build +- Fixed travis build, testing with 2.2, 2.3, 2.4, 2.5 and 2.6 of Ruby. Right now only 2.2 and 2.3 pass. ## [5.2.0] - 2019-03-11 From f8ddc5b8eae6ca880289f1e69e58c2100e5b5456 Mon Sep 17 00:00:00 2001 From: Scott Rohde Date: Thu, 8 Aug 2019 16:24:20 -0500 Subject: [PATCH 31/36] In get_insertion_data, change the way interactively_specified_values is set. In Ruby >= 2.4, we can no longer rely on @session["global_values"].clone throwing an exception when @session["global_values"] is nil so we must explicitely check whether @session has the key :global_values. --- app/models/bulk_upload_data_set.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/bulk_upload_data_set.rb b/app/models/bulk_upload_data_set.rb index 17a375a8d..777ec0dee 100644 --- a/app/models/bulk_upload_data_set.rb +++ b/app/models/bulk_upload_data_set.rb @@ -1900,7 +1900,9 @@ def get_insertion_data # Get interactively-specified values, or set to empty hash if nil; since we # are going to alter interactively_specified_values, we use clone to make a # copy so that the session value remains as is. - interactively_specified_values = @session["global_values"].clone rescue ActionController::Parameters.new + interactively_specified_values = @session.has_key?(:global_values) \ + ? @session[:global_values].clone \ + : ActionController::Parameters.new # TO DO: decide if this code serves any useful purpose: # Double-check that all form fields are were non-empty: From 2bd345b49edbe86764856558de299682d5d80fa2 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Fri, 9 Aug 2019 09:53:13 -0500 Subject: [PATCH 32/36] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0ec47bfd..c84876501 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,8 @@ Currently BETY does not compile correctly with any newer version of Ruby than 2. - Updated GEM list to fix security fixes as recommended by dependbot on GitHub. - Restores method assignment in Bulk Uploads -- Fixed travis build, testing with 2.2, 2.3, 2.4, 2.5 and 2.6 of Ruby. Right now only 2.2 and 2.3 pass. +- Fixed travis build, testing with 2.2, 2.3, 2.4, 2.5 and 2.6 of Ruby. +- Fixed errors when running BETY with 2.4 and 2.5 ## [5.2.0] - 2019-03-11 From 5979854e5ae946a5a63f32160f1edd1665bb2e61 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Fri, 9 Aug 2019 09:55:21 -0500 Subject: [PATCH 33/36] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c84876501..21ac810ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ For more information about this file see also [Keep a Changelog](http://keepacha ## [5.2.1] - 2019-08-07 -Currently BETY does not compile correctly with any newer version of Ruby than 2.3. We are aware of this issue and tracking it #664. +Currently the tested and recommended version of Ruby to be used with BETY is 2.3. BETY compiles and passes tests with 2.4 and 2.5 but have not been tested thoroughly. Currently BETY does not compile with Ruby 2.6 (see #667). ### Added From e55b2cb3176e784c1cecc25955b2f9b9baaf9c69 Mon Sep 17 00:00:00 2001 From: Scott Rohde Date: Fri, 9 Aug 2019 11:24:31 -0500 Subject: [PATCH 34/36] Make the citation controller's "show" action throw an ActiveRecord::RecordNotFound exception when the given model is not found, just as all the other controllers do. This fixes issue #668. --- app/controllers/citations_controller.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/controllers/citations_controller.rb b/app/controllers/citations_controller.rb index 415e49973..46d2338e0 100644 --- a/app/controllers/citations_controller.rb +++ b/app/controllers/citations_controller.rb @@ -123,7 +123,10 @@ def index # GET /citations/1 # GET /citations/1.xml def show - @citation = Citation.where(:id => params[:id]).includes(params[:include]).first + # find_by! throws an ActiveRecord::RecordNotFound exception if no citation + # with the given id exists so that we don't attempt to display a nil + # citation. + @citation = Citation.includes(params[:include]).find_by!(:id => params[:id]) respond_to do |format| format.html # show.html.erb From 4fe4a453973390bff20559da5e12c297498f83b6 Mon Sep 17 00:00:00 2001 From: Scott Rohde Date: Fri, 9 Aug 2019 15:10:46 -0500 Subject: [PATCH 35/36] Skip the test for blank lines in CSV files since these are now parsed without error in Ruby 2.6. Also, fixed a capybara-webkit based test which started failing regularly under Ruby 2.6. --- spec/controllers/bulk_upload_controller_spec.rb | 5 ++++- spec/features/site_integration_spec.rb | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/spec/controllers/bulk_upload_controller_spec.rb b/spec/controllers/bulk_upload_controller_spec.rb index 2dc77c361..4f10db5fc 100644 --- a/spec/controllers/bulk_upload_controller_spec.rb +++ b/spec/controllers/bulk_upload_controller_spec.rb @@ -119,7 +119,10 @@ class BulkUploadController # TODO: possibly test various kinds of invalid files and what messages result context "uploading an invalid csv file" do - it "should throw an error and redirect to the start_upload page" do + # This test tests for blank lines in the CSV file, which are no longer + # invalid in Ruby 2.6. So we skip this test but keep it around for + # documentation. + it "should throw an error and redirect to the start_upload page", skip: true do @file = fixture_file_upload("/files/bulk_upload/invalid_file.csv", "text/csv") @form = { 'new upload' => true, "CSV file" => @file } diff --git a/spec/features/site_integration_spec.rb b/spec/features/site_integration_spec.rb index cd488431d..32ab52ab3 100644 --- a/spec/features/site_integration_spec.rb +++ b/spec/features/site_integration_spec.rb @@ -56,7 +56,9 @@ it 'show allow adding new related citations', js: true do click_link 'View Related Citations' fill_in 'search_citations', with: 'Wood' - sleep 2 # 1 often isn't enough + # In Ruby 2.6 especially, merely filling in a field doesn't seem to + # trigger the needed keyup event, so do it manually: + find_field('search_citations').trigger(:keyup) click_link '+' click_button 'Update' # reopen related citations listing From 4b5453ea1c5c95869173770e2d01f3faa5f81f17 Mon Sep 17 00:00:00 2001 From: Rob Kooper Date: Sat, 10 Aug 2019 13:38:43 -0500 Subject: [PATCH 36/36] Update CHANGELOG.md --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21ac810ee..aa78d4db6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ For more information about this file see also [Keep a Changelog](http://keepacha ## [5.2.1] - 2019-08-07 -Currently the tested and recommended version of Ruby to be used with BETY is 2.3. BETY compiles and passes tests with 2.4 and 2.5 but have not been tested thoroughly. Currently BETY does not compile with Ruby 2.6 (see #667). +Currently the tested and recommended version of Ruby to be used with BETY is 2.3. BETY compiles and passes tests with 2.4, 2.5 and 2.6 but have not been tested thoroughly with these versions. ### Added @@ -24,7 +24,8 @@ Currently the tested and recommended version of Ruby to be used with BETY is 2.3 - Updated GEM list to fix security fixes as recommended by dependbot on GitHub. - Restores method assignment in Bulk Uploads - Fixed travis build, testing with 2.2, 2.3, 2.4, 2.5 and 2.6 of Ruby. -- Fixed errors when running BETY with 2.4 and 2.5 +- Fixed errors when running BETY with 2.4, 2.5 and 2.6 +- #668 : show "not found" page for invalid citations. ## [5.2.0] - 2019-03-11