From d9bf22cb3b2a27534b64b17ad9e962710c886849 Mon Sep 17 00:00:00 2001 From: Gene Hoffman <30377676+hoffmang9@users.noreply.github.com> Date: Wed, 25 Dec 2019 11:18:07 -0800 Subject: [PATCH] Separate timelord, add Windows/CentOS/AMZN (#62) 1.1.1 release * windows build instructions * install.sh newline fix \n wasn't as cross platform as it should be so reverted to an extra echo * Add CentOS to README.md Full CentOS 7.7 install instructions * mention blspy * confirmed boost 1.67.0 works on Ubuntu * Update README.md. * Fix problem with zip download. * Remove scm warning Revert install instructions due to scm fix * Update README and install scripts to separate timelord installation Do not install VDF server and its dependencies in install.sh, create a separate script install_timelord.sh for that. * Move installation of VDF server dependencies to install_timelord.sh * First and broken attempt at CentOS install * Working CentOS 7.7 install * typos in timelord explanation * Timlord/VDF build instructions Specifics for CentOS and MacOS * Various install tips and tricks * spelling error * Add AMZN linux 2 install * Add AMZN linux 2 timelord install Also clean up base install instructions * Combine TL for AMZN and CentOS * Nits for AMZN and CentOS * CentOS, AMZN timelord paralleling * Fix header layouts * Replace VDF server compilation scripts with Makefile * layout improvements * fast_vdf: Remove unused files * Partial ARM install support * missing l in lib * Fleshed out Windows/WSL installs And verified the method for upgrade to 19.x * Move MacOS install fullnode to bottom * Change to mongo "system-wide" * Add mongo start to WSL * Add actual upgrade for WSL/Ubuntu Co-authored-by: Mariano Sorgente Co-authored-by: Rostislav Skudnov Co-authored-by: Richard Kiss Co-authored-by: wjblanke --- README.md | 227 +++++++++++++++++++++++--- install.sh | 19 +-- install_timelord.sh | 27 +++ lib/chiavdf/fast_vdf/Makefile | 36 ++++ lib/chiavdf/fast_vdf/compile.sh | 2 - lib/chiavdf/fast_vdf/copy_to_repo.sh | 2 - lib/chiavdf/fast_vdf/install.sh | 2 - lib/chiavdf/fast_vdf/install_child.sh | 35 ---- lib/chiavdf/fast_vdf/run.sh | 2 - lib/chiavdf/fast_vdf/sconstruct | 24 --- lib/chiavdf/fast_vdf/upload.sh | 2 - lib/chiavdf/fast_vdf/vdf_new.cpp | 30 ---- 12 files changed, 273 insertions(+), 135 deletions(-) create mode 100644 install_timelord.sh create mode 100644 lib/chiavdf/fast_vdf/Makefile delete mode 100644 lib/chiavdf/fast_vdf/compile.sh delete mode 100755 lib/chiavdf/fast_vdf/copy_to_repo.sh delete mode 100755 lib/chiavdf/fast_vdf/install.sh delete mode 100755 lib/chiavdf/fast_vdf/install_child.sh delete mode 100755 lib/chiavdf/fast_vdf/run.sh delete mode 100755 lib/chiavdf/fast_vdf/sconstruct delete mode 100755 lib/chiavdf/fast_vdf/upload.sh delete mode 100644 lib/chiavdf/fast_vdf/vdf_new.cpp diff --git a/README.md b/README.md index d5aee3270b5e..163316040689 100644 --- a/README.md +++ b/README.md @@ -3,51 +3,224 @@ Python 3.7 is used for this project. Make sure your default python version is >= You will need to enable [UPnP](https://www.homenethowto.com/ports-and-nat/upnp-automatic-port-forward/) on your router or add a NAT (for IPv4 but not IPv6) and firewall rule to allow TCP port 8444 access to your peer. These methods tend to be router make/model specific. -### Install on Debian/Ubuntu +For alpha testnet most should only install harvesters, farmers, plotter and full nodes. Building timelords and VDFs is for sophisticated users in most environments. Chia Network and additional volunteers are running sufficient time lords for testnet consensus. + +## Install harvester, farmer, plotter, and full node + +### Debian/Ubuntu ```bash sudo apt-get update -sudo apt-get install build-essential cmake python3-dev python3-venv --no-install-recommends +sudo apt-get install build-essential cmake python3-dev python3-venv libssl-dev libffi-dev --no-install-recommends + +git clone https://github.com/Chia-Network/chia-blockchain.git +cd chia-blockchain + +sh install.sh + +# Install MongoDB Community Edition +# Instructions - https://docs.mongodb.com/manual/administration/install-on-linux/ + +# Run mongo database if not running system-wide +mongod --fork --dbpath ./db/ --logpath mongod.log + +. .venv/bin/activate +``` +### Amazon Linux 2 + +```bash +sudo yum update +sudo yum install gcc-c++ cmake3 wget git openssl openssl-devel +sudo yum install python3 python3-devel libffi-devel + +# CMake - add a symlink for cmake3 - required by blspy +sudo ln -s /usr/bin/cmake3 /usr/local/bin/cmake + +git clone https://github.com/Chia-Network/chia-blockchain.git +cd chia-blockchain + +sh install.sh + +# Install MongoDB Community Edition +# Instructions - https://docs.mongodb.com/manual/administration/install-on-linux/ -# Update boost version to 1.71.0 or greater if needed, check version: dpkg -s libboost-dev | grep 'Version' -# Install from www.boost.org +# Run mongo database if not running system-wide +mongod --fork --dbpath ./db/ --logpath mongod.log + +. .venv/bin/activate +``` +### CentOS 7 + +```bash +sudo yum update +sudo yum install centos-release-scl-rh epel-release +sudo yum install devtoolset-8-toolchain cmake3 libffi-devel +sudo yum install wget git openssl openssl-devel + +# CMake - add a symlink for cmake3 - required by blspy +sudo ln -s /usr/bin/cmake3 /usr/local/bin/cmake + +scl enable devtoolset-8 bash + +# Install Python 3.7.5 (current rpm's are 3.6.x) +wget https://www.python.org/ftp/python/3.7.5/Python-3.7.5.tgz +tar -zxvf Python-3.7.5.tgz; cd Python-3.7.5 +./configure --enable-optimizations; sudo make install; cd .. + +git clone https://github.com/Chia-Network/chia-blockchain.git +cd chia-blockchain sh install.sh # Install MongoDB Community Edition # Instructions - https://docs.mongodb.com/manual/administration/install-on-linux/ -# Run mongo database +# Run mongo database if not running system-wide +mongod --fork --dbpath ./db/ --logpath mongod.log + +. .venv/bin/activate +``` + +### Windows (WSL + Ubuntu) +#### Install WSL + Ubuntu 18.04 LTS, upgrade to Ubuntu 19.x + +This will require multiple reboots. From an Administrator PowerShell +`Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux` +and then +`Enable-WindowsOptionalFeature -Online -FeatureName VirtualMachinePlatform`. +Once that is complete, install Ubuntu 18.04 LTS from the Windows Store. +```bash +# Upgrade to 19.x +sudo nano /etc/update-manager/release-upgrades +# Change "Prompt=lts" to "Prompt=normal" save and exit + +sudo apt-get -y update +sudo apt-get -y upgrade +sudo do-release-upgrade + +sudo apt-get install -y build-essential cmake python3-dev python3-venv mongodb software-properties-common --no-install-recommends + +git clone https://github.com/Chia-Network/chia-blockchain.git +cd chia-blockchain + +sudo sh install.sh + +# Run mongo database if not running system-wide mongod --fork --dbpath ./db/ --logpath mongod.log . .venv/bin/activate ``` -### Install on MacOS + +#### Alternate method for Ubuntu 18.04 LTS +In `./install.sh`: +Change `python3` to `python3.7` +Each line that starts with `pip ...` becomes `python -m pip ...` + +```bash +sudo apt-get -y update +sudo apt-get install -y build-essential cmake python3-dev python3-venv mongodb software-properties-common --no-install-recommends + +# Install python3.7 with ppa +sudo add-apt-repository -y ppa:deadsnakes/ppa +sudo apt-get -y update +sudo apt-get install -y python3.7 python3.7-venv python3.7-dev + +git clone https://github.com/Chia-Network/chia-blockchain.git +cd chia-blockchain + +sudo sh install.sh + +# Run mongo database if not running system-wide +mongod --fork --dbpath ./db/ --logpath mongod.log + +. .venv/bin/activate +``` + +### MacOS Make sure [brew](https://brew.sh/) is available before starting the setup. ```bash brew tap mongodb/brew brew upgrade python -brew install cmake boost gmp mpir mpfr mongodb-community@4.2 - -git clone https://github.com/Chia-Network/chia-blockchain.git && cd chia-blockchain +brew install cmake mongodb-community@4.2 -git clone https://github.com/wbhart/flint2 +git clone https://github.com/Chia-Network/chia-blockchain.git +cd chia-blockchain sh install.sh -# Run mongo database +# Run mongo database if not running system-wide mongod --fork --dbpath ./db/ --logpath mongod.log . .venv/bin/activate ``` -### Generate keys + +## Install timelord +Note: this step is needed only if you intend to run a timelord or a local simulation. +These assume you've already successfully installed harvester, farmer, plotting, and full node above. +### Ubuntu/Debian +```bash +cd chia-blockchain + +sh install_timelord.sh +``` +### Amazon Linux 2 and CentOS 7 +```bash +#Only for Amazon Linux 2 +sudo amazon-linux-extras install epel + +sudo yum install gmp-devel mpfr-devel + +# Install Boost 1.72.0 +wget https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.gz +tar -zxvf boost_1_72_0.tar.gz +cd boost_1_72_0 +./bootstrap.sh --prefix=/usr/local +sudo ./b2 install --prefix=/usr/local --with=all; cd .. + +# Install Flint2 +git clone https://github.com/wbhart/flint2 +cd flint2; ./configure; sudo make install; cd .. +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib + +cd chia-blockchain + +sh install_timelord.sh +``` + +### Windows (WSL + Ubuntu) +#### Install WSL + Ubuntu upgraded to 19.x +```bash +cd chia-blockchain + +sh install_timelord.sh +``` +#### Alternate method for Ubuntu 18.04 +```bash +# Install boost 1.70 with ppa +sudo add-apt-repository -y ppa:mhier/libboost-latest +sudo apt-get update +sudo apt-get install libboost1.70 libboost1.70-dev +``` + +### MacOS +```bash +brew install boost gmp mpir mpfr + +cd chia-blockchain + +git clone https://github.com/wbhart/flint2 + +sh install_timelord.sh +``` + +## Generate keys First, create some keys by running the following script: ```bash python -m scripts.regenerate_keys ``` -### Run a full node +## Run a full node To run a full node on port 8002, and connect to the testnet, run the following command. This wil also start an ssh server in port 8222 for the UI, which you can connect to to see the state of the node. @@ -56,7 +229,7 @@ python -m src.server.start_full_node "127.0.0.1" 8444 -id 1 -u 8222 & ssh -p 8222 localhost ``` -### Run a farmer + full node +## Run a farmer + full node Farmers are entities in the network who use their hard drive space to try to create blocks (like Bitcoin's miners), and earn block rewards. First, you must generate some hard drive plots, which can take a long time depending on the size of the plots (the k variable). Then, run the farmer + full node with @@ -66,24 +239,34 @@ python -m scripts.create_plots -k 20 -n 10 sh ./scripts/run_farming.sh ``` -### Run a timelord + full node +## Run a timelord + full node Timelords execute sequential verifiable delay functions (proofs of time), that get added to -blocks to make them valid. This requires fast CPUs and a lot of memory. +blocks to make them valid. This requires fast CPUs and a lot of memory as well as completing +both install steps above. ```bash sh ./scripts/run_timelord.sh ``` -### Tips +## Tips When running the servers on Mac OS, allow the application to accept incoming connections. -UPnP is enabled by default, to open the port for incoming connections. If this causes issues, you can disable it in the configuration. Some routers may require port forwarding, or enabling UPnP in the router configuration. +Ubuntu 19.xx, Amazon Linux 2, and CentOS 7.7 or newer are the easiest linux install environments currently. + +UPnP is enabled by default, to open the port for incoming connections. If this causes issues, +you can disable it in the configuration. Some routers may require port forwarding, or enabling +UPnP in the router configuration. -Due to the nature of proof of space lookups by the harvester you should limit the number of plots on a physical drive to 50 or less. +Due to the nature of proof of space lookups by the harvester you should limit the number of plots +on a physical drive to 50 or less. This limit should significantly increase before beta. You can also run the simulation, which runs all servers and multiple full nodes, locally, at once. -If you want to run the simulation, change the introducer ip in ./config/config.yaml so that the full node points to the local introducer (127.0.0.1:8445). -Note the the simulation is local only. -The introducer will only know the local ips of the full nodes, so it cannot broadcast the correct ips to external peers. +If you want to run the simulation, change the introducer ip in ./config/config.yaml so that the +full node points to the local introducer (127.0.0.1:8445). + +Note the the simulation is local only and requires installation of timelords and VDFs. + +The introducer will only know the local ips of the full nodes, so it cannot broadcast the correct +ips to external peers. ```bash sh ./scripts/run_all_simulation.sh diff --git a/install.sh b/install.sh index f84b01d53ca1..65eddb5f53b9 100644 --- a/install.sh +++ b/install.sh @@ -1,22 +1,13 @@ +#!/bin/bash +set -e + python3 -m venv .venv . .venv/bin/activate pip install wheel # For building blspy pip install -e . pip install -r requirements.txt -# Install libgmp, libboost, and libflint, and then run the following -# Check for git clone of flint2 on MacOS and install if found -if [ -f flint2/configure ]; then - cd flint2/ - if [ ! -f Makefile ]; then - ./configure - fi - make -j4 - make install - cd ../ -fi - -cd lib/chiavdf/fast_vdf && sh install.sh -echo "\nChia blockchain install.sh complete." +echo "" +echo "Chia blockchain install.sh complete." echo "For assistance join us on Keybase in the #testnet chat channel" echo "https://keybase.io/team/chia_network.public" diff --git a/install_timelord.sh b/install_timelord.sh new file mode 100644 index 000000000000..4b836944f993 --- /dev/null +++ b/install_timelord.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + +if [ `uname` = "Linux" ] && type apt-get; then + # Debian/Ubuntu + sudo apt-get install -y libgmp3-dev libflint-dev \ + libboost-dev libboost-system-dev +fi + +# Install libgmp, libboost, and libflint, and then run the following +# Check for git clone of flint2 on MacOS and install if found +if [ -f flint2/configure ]; then + cd flint2/ + if [ ! -f Makefile ]; then + ./configure + fi + make -j4 + make install + cd ../ +fi + +make -C lib/chiavdf/fast_vdf + +echo "" +echo "Chia blockchain install_timelord.sh complete." +echo "For assistance join us on Keybase in the #testnet chat channel" +echo "https://keybase.io/team/chia_network.public" diff --git a/lib/chiavdf/fast_vdf/Makefile b/lib/chiavdf/fast_vdf/Makefile new file mode 100644 index 000000000000..8ea71905dc52 --- /dev/null +++ b/lib/chiavdf/fast_vdf/Makefile @@ -0,0 +1,36 @@ +UNAME := $(shell uname) + +ifeq ($(UNAME),Linux) +ALL_INSTR := $(shell grep -w avx2 /proc/cpuinfo | grep -w fma | grep -w -q adx \ + && echo 1 || echo 0) +else +ALL_INSTR := 0 +endif + +LDFLAGS += -no-pie +LDLIBS += -lgmpxx -lgmp -lflint -lboost_system -pthread +CXXFLAGS += -std=c++1z -D VDF_MODE=0 -D ENABLE_ALL_INSTRUCTIONS=$(ALL_INSTR) \ + -no-pie -pthread -march=native +ifeq ($(UNAME),Darwin) +CXXFLAGS += -D CHIAOSX=1 +else +OPT_CFLAGS = -O3 +endif + +.PHONY: all clean + +all: vdf_server + +clean: + rm -f *.o vdf_server compile_asm + +vdf_server: vdf_server.o asm_compiled.o + $(CXX) $(LDFLAGS) -o $@ $^ $(LDLIBS) + +vdf_server.o: CXXFLAGS += $(OPT_CFLAGS) + +asm_compiled.s: compile_asm + ./compile_asm + +compile_asm: compile_asm.o + $(CXX) $(LDFLAGS) -o $@ $^ $(LDLIBS) diff --git a/lib/chiavdf/fast_vdf/compile.sh b/lib/chiavdf/fast_vdf/compile.sh deleted file mode 100644 index cc10890e868b..000000000000 --- a/lib/chiavdf/fast_vdf/compile.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -cat /proc/cpuinfo | grep -w cmovf | grep -w -q avx diff --git a/lib/chiavdf/fast_vdf/copy_to_repo.sh b/lib/chiavdf/fast_vdf/copy_to_repo.sh deleted file mode 100755 index 5471159ddb69..000000000000 --- a/lib/chiavdf/fast_vdf/copy_to_repo.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -cp *.c *.cpp *.h *.sh sconstruct ~/projects/chia_vdf_entry/entry/ diff --git a/lib/chiavdf/fast_vdf/install.sh b/lib/chiavdf/fast_vdf/install.sh deleted file mode 100755 index e8615ce5ae95..000000000000 --- a/lib/chiavdf/fast_vdf/install.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -./install_child.sh 2>&1 diff --git a/lib/chiavdf/fast_vdf/install_child.sh b/lib/chiavdf/fast_vdf/install_child.sh deleted file mode 100755 index 5583993d2a80..000000000000 --- a/lib/chiavdf/fast_vdf/install_child.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -set -v - -cat /proc/cpuinfo | grep -e MHz -e GHz -cat /proc/cpuinfo | grep flags | head -n 1 -enable_all_instructions=0 -if cat /proc/cpuinfo | grep -w avx2 | grep -w fma | grep -w -q adx; then - enable_all_instructions=1 -fi -echo "enable_all_instructions: $enable_all_instructions" - -if [[ "$OSTYPE" == "linux-gnu" ]]; then - sudo apt-get install libgmp3-dev -y - sudo apt-get install libflint-dev -y - sudo apt install libboost-all-dev - opt_flags="-O3" - link_flags="-no-pie -lgmpxx -lgmp -lflint -lpthread -lboost_system" - compile_flags="-std=c++1z -D VDF_MODE=0 -D ENABLE_ALL_INSTRUCTIONS=$enable_all_instructions -no-pie -march=native" -elif [[ "$OSTYPE" == "darwin"* ]]; then - opt_flags="" # Catalina clang issues - link_flags="-no-pie -lgmpxx -lgmp -lflint" - compile_flags="-std=c++1z -D CHIAOSX=1 -D VDF_MODE=0 -D ENABLE_ALL_INSTRUCTIONS=$enable_all_instructions -no-pie -march=native" -else - opt_flags="-O3" - link_flags="-no-pie -lgmpxx -lgmp -lflint" - compile_flags="-std=c++1z -D VDF_MODE=0 -D ENABLE_ALL_INSTRUCTIONS=$enable_all_instructions -no-pie -march=native" -fi - - -g++ -o compile_asm.o -c compile_asm.cpp $compile_flags -O0 -g++ -o compile_asm compile_asm.o $link_flags -./compile_asm -as -o asm_compiled.o asm_compiled.s -g++ -o vdf_server.o -c vdf_server.cpp $compile_flags $opt_flags -g++ -o vdf_server vdf_server.o asm_compiled.o $link_flags diff --git a/lib/chiavdf/fast_vdf/run.sh b/lib/chiavdf/fast_vdf/run.sh deleted file mode 100755 index 7a870d71b70d..000000000000 --- a/lib/chiavdf/fast_vdf/run.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -./vdf $1 $2 diff --git a/lib/chiavdf/fast_vdf/sconstruct b/lib/chiavdf/fast_vdf/sconstruct deleted file mode 100755 index a0fb50b9c623..000000000000 --- a/lib/chiavdf/fast_vdf/sconstruct +++ /dev/null @@ -1,24 +0,0 @@ -import gch - -ccflags=' -O0' -#ccflags=' -O3' - -ccflags = '-D VDF_MODE=1 -D ENABLE_ALL_INSTRUCTIONS=0 -no-pie -march=native' + ccflags - -env.Append( - CCFLAGS=ccflags, - LINKFLAGS= '-no-pie', - LIBS=['gmpxx', 'gmp', 'flint', 'pthread'] -); - -gch.generate(env); - -env['precompiled_header']=File('include.h'); -env['Gch']=env.Gch(target='include.h.gch', source=env['precompiled_header']); - -#env.Program('gcd_test.cpp'); -#env.Program('vdf.cpp'); - -env.Program( 'compile_asm', 'compile_asm.cpp', CCFLAGS = ccflags + ' -O0' ); -env.Command( 'asm_compiled.s', 'compile_asm', "./compile_asm" ); -env.Program( 'vdf', [ 'vdf.cpp', 'asm_compiled.s' ] ); \ No newline at end of file diff --git a/lib/chiavdf/fast_vdf/upload.sh b/lib/chiavdf/fast_vdf/upload.sh deleted file mode 100755 index 0b538e45707b..000000000000 --- a/lib/chiavdf/fast_vdf/upload.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -scp *.c *.cpp *.h *.sh sconstruct $VM:projects/chia_vdf/ diff --git a/lib/chiavdf/fast_vdf/vdf_new.cpp b/lib/chiavdf/fast_vdf/vdf_new.cpp deleted file mode 100644 index 1a8632d4cb75..000000000000 --- a/lib/chiavdf/fast_vdf/vdf_new.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/*#include "include.h" - -#include "integer.h" - -#include "vdf_new.h" - -int main(int argc, char** argv) { - parse_args(argc, argv); - - integer a; - integer b; - integer c; - generator_for_discriminant(arg_discriminant, a, b, c); - - for (int x=0;x